Merge pull request #2 from PaddlePaddle/develop

mergepaddleocr

deploy/lite/db_post_process.h

+// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#pragma once
+
+#include <math.h>
+
+#include <iostream>
+#include <map>
+#include <vector>
+
+#include "clipper.hpp"
+#include "opencv2/core.hpp"
+#include "opencv2/imgcodecs.hpp"
+#include "opencv2/imgproc.hpp"
+
+template <class T> T clamp(T x, T min, T max) {
+  if (x > max)
+    return max;
+  if (x < min)
+    return min;
+  return x;
+}
+
+std::vector<std::vector<float>> Mat2Vector(cv::Mat mat);
+
+void GetContourArea(std::vector<std::vector<float>> box, float unclip_ratio,
+                    float &distance);
+
+cv::RotatedRect Unclip(std::vector<std::vector<float>> box, float unclip_ratio);
+
+std::vector<std::vector<float>> Mat2Vector(cv::Mat mat);
+
+bool XsortFp32(std::vector<float> a, std::vector<float> b);
+
+bool XsortInt(std::vector<int> a, std::vector<int> b);
+
+std::vector<std::vector<int>>
+OrderPointsClockwise(std::vector<std::vector<int>> pts);
+
+std::vector<std::vector<float>> GetMiniBoxes(cv::RotatedRect box, float &ssid);
+
+float BoxScoreFast(std::vector<std::vector<float>> box_array, cv::Mat pred);
+
+std::vector<std::vector<std::vector<int>>>
+BoxesFromBitmap(const cv::Mat pred, const cv::Mat bitmap,
+                std::map<std::string, double> Config);
+
+std::vector<std::vector<std::vector<int>>>
+FilterTagDetRes(std::vector<std::vector<std::vector<int>>> boxes, float ratio_h,
+                float ratio_w, cv::Mat srcimg);

deploy/lite/ocr_db_crnn.cc

+// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+#include "paddle_api.h" // NOLINT
+#include <chrono>
+
+#include "crnn_process.h"
+#include "db_post_process.h"
+
+using namespace paddle::lite_api; // NOLINT
+using namespace std;
+
+// fill tensor with mean and scale and trans layout: nhwc -> nchw, neon speed up
+void NeonMeanScale(const float *din, float *dout, int size,
+                   const std::vector<float> mean,
+                   const std::vector<float> scale) {
+  if (mean.size() != 3 || scale.size() != 3) {
+    std::cerr << "[ERROR] mean or scale size must equal to 3\n";
+    exit(1);
+  }
+  float32x4_t vmean0 = vdupq_n_f32(mean[0]);
+  float32x4_t vmean1 = vdupq_n_f32(mean[1]);
+  float32x4_t vmean2 = vdupq_n_f32(mean[2]);
+  float32x4_t vscale0 = vdupq_n_f32(scale[0]);
+  float32x4_t vscale1 = vdupq_n_f32(scale[1]);
+  float32x4_t vscale2 = vdupq_n_f32(scale[2]);
+
+  float *dout_c0 = dout;
+  float *dout_c1 = dout + size;
+  float *dout_c2 = dout + size * 2;
+
+  int i = 0;
+  for (; i < size - 3; i += 4) {
+    float32x4x3_t vin3 = vld3q_f32(din);
+    float32x4_t vsub0 = vsubq_f32(vin3.val[0], vmean0);
+    float32x4_t vsub1 = vsubq_f32(vin3.val[1], vmean1);
+    float32x4_t vsub2 = vsubq_f32(vin3.val[2], vmean2);
+    float32x4_t vs0 = vmulq_f32(vsub0, vscale0);
+    float32x4_t vs1 = vmulq_f32(vsub1, vscale1);
+    float32x4_t vs2 = vmulq_f32(vsub2, vscale2);
+    vst1q_f32(dout_c0, vs0);
+    vst1q_f32(dout_c1, vs1);
+    vst1q_f32(dout_c2, vs2);
+
+    din += 12;
+    dout_c0 += 4;
+    dout_c1 += 4;
+    dout_c2 += 4;
+  }
+  for (; i < size; i++) {
+    *(dout_c0++) = (*(din++) - mean[0]) * scale[0];
+    *(dout_c1++) = (*(din++) - mean[1]) * scale[1];
+    *(dout_c2++) = (*(din++) - mean[2]) * scale[2];
+  }
+}
+
+// resize image to a size multiple of 32 which is required by the network
+cv::Mat DetResizeImg(const cv::Mat img, int max_size_len,
+                     std::vector<float> &ratio_hw) {
+  int w = img.cols;
+  int h = img.rows;
+
+  float ratio = 1.f;
+  int max_wh = w >= h ? w : h;
+  if (max_wh > max_size_len) {
+    if (h > w) {
+      ratio = static_cast<float>(max_size_len) / static_cast<float>(h);
+    } else {
+      ratio = static_cast<float>(max_size_len) / static_cast<float>(w);
+    }
+  }
+
+  int resize_h = static_cast<int>(float(h) * ratio);
+  int resize_w = static_cast<int>(float(w) * ratio);
+  if (resize_h % 32 == 0)
+    resize_h = resize_h;
+  else if (resize_h / 32 < 1 + 1e-5)
+    resize_h = 32;
+  else
+    resize_h = (resize_h / 32 - 1) * 32;
+
+  if (resize_w % 32 == 0)
+    resize_w = resize_w;
+  else if (resize_w / 32 < 1 + 1e-5)
+    resize_w = 32;
+  else
+    resize_w = (resize_w / 32 - 1) * 32;
+
+  cv::Mat resize_img;
+  cv::resize(img, resize_img, cv::Size(resize_w, resize_h));
+
+  ratio_hw.push_back(static_cast<float>(resize_h) / static_cast<float>(h));
+  ratio_hw.push_back(static_cast<float>(resize_w) / static_cast<float>(w));
+  return resize_img;
+}
+
+void RunRecModel(std::vector<std::vector<std::vector<int>>> boxes, cv::Mat img,
+                 std::shared_ptr<PaddlePredictor> predictor_crnn,
+                 std::vector<std::string> &rec_text,
+                 std::vector<float> &rec_text_score,
+                 std::vector<std::string> charactor_dict) {
+  std::vector<float> mean = {0.5f, 0.5f, 0.5f};
+  std::vector<float> scale = {1 / 0.5f, 1 / 0.5f, 1 / 0.5f};
+
+  cv::Mat srcimg;
+  img.copyTo(srcimg);
+  cv::Mat crop_img;
+  cv::Mat resize_img;
+
+  int index = 0;
+  for (int i = boxes.size() - 1; i >= 0; i--) {
+    crop_img = GetRotateCropImage(srcimg, boxes[i]);
+    float wh_ratio =
+        static_cast<float>(crop_img.cols) / static_cast<float>(crop_img.rows);
+
+    resize_img = CrnnResizeImg(crop_img, wh_ratio);
+    resize_img.convertTo(resize_img, CV_32FC3, 1 / 255.f);
+
+    const float *dimg = reinterpret_cast<const float *>(resize_img.data);
+
+    std::unique_ptr<Tensor> input_tensor0(
+        std::move(predictor_crnn->GetInput(0)));
+    input_tensor0->Resize({1, 3, resize_img.rows, resize_img.cols});
+    auto *data0 = input_tensor0->mutable_data<float>();
+
+    NeonMeanScale(dimg, data0, resize_img.rows * resize_img.cols, mean, scale);
+    //// Run CRNN predictor
+    predictor_crnn->Run();
+
+    // Get output and run postprocess
+    std::unique_ptr<const Tensor> output_tensor0(
+        std::move(predictor_crnn->GetOutput(0)));
+    auto *rec_idx = output_tensor0->data<int64>();
+
+    auto rec_idx_lod = output_tensor0->lod();
+    auto shape_out = output_tensor0->shape();
+
+    std::vector<int> pred_idx;
+    for (int n = static_cast<int>(rec_idx_lod[0][0]);
+         n < static_cast<int>(rec_idx_lod[0][1]); n += 1) {
+      pred_idx.push_back(static_cast<int>(rec_idx[n]));
+    }
+
+    if (pred_idx.size() < 1e-3)
+      continue;
+
+    index += 1;
+    std::string pred_txt = "";
+    for (int n = 0; n < pred_idx.size(); n++) {
+      pred_txt += charactor_dict[pred_idx[n]];
+    }
+    rec_text.push_back(pred_txt);
+
+    ////get score
+    std::unique_ptr<const Tensor> output_tensor1(
+        std::move(predictor_crnn->GetOutput(1)));
+    auto *predict_batch = output_tensor1->data<float>();
+    auto predict_shape = output_tensor1->shape();
+
+    auto predict_lod = output_tensor1->lod();
+
+    int blank = predict_shape[1];
+    float score = 0.f;
+    int count = 0;
+
+    for (int n = predict_lod[0][0]; n < predict_lod[0][1] - 1; n++) {
+      int argmax_idx =
+          static_cast<int>(Argmax(&predict_batch[n * predict_shape[1]],
+                                  &predict_batch[(n + 1) * predict_shape[1]]));
+      float max_value =
+          float(*std::max_element(&predict_batch[n * predict_shape[1]],
+                                  &predict_batch[(n + 1) * predict_shape[1]]));
+
+      if (blank - 1 - argmax_idx > 1e-5) {
+        score += max_value;
+        count += 1;
+      }
+    }
+    score /= count;
+    rec_text_score.push_back(score);
+  }
+}
+
+std::vector<std::vector<std::vector<int>>>
+RunDetModel(std::shared_ptr<PaddlePredictor> predictor, cv::Mat img,
+            std::map<std::string, double> Config) {
+  // Read img
+  int max_side_len = int(Config["max_side_len"]);
+
+  cv::Mat srcimg;
+  img.copyTo(srcimg);
+
+  std::vector<float> ratio_hw;
+  img = DetResizeImg(img, max_side_len, ratio_hw);
+  cv::Mat img_fp;
+  img.convertTo(img_fp, CV_32FC3, 1.0 / 255.f);
+
+  // Prepare input data from image
+  std::unique_ptr<Tensor> input_tensor0(std::move(predictor->GetInput(0)));
+  input_tensor0->Resize({1, 3, img_fp.rows, img_fp.cols});
+  auto *data0 = input_tensor0->mutable_data<float>();
+
+  std::vector<float> mean = {0.485f, 0.456f, 0.406f};
+  std::vector<float> scale = {1 / 0.229f, 1 / 0.224f, 1 / 0.225f};
+  const float *dimg = reinterpret_cast<const float *>(img_fp.data);
+  NeonMeanScale(dimg, data0, img_fp.rows * img_fp.cols, mean, scale);
+
+  // Run predictor
+  predictor->Run();
+
+  // Get output and post process
+  std::unique_ptr<const Tensor> output_tensor(
+      std::move(predictor->GetOutput(0)));
+  auto *outptr = output_tensor->data<float>();
+  auto shape_out = output_tensor->shape();
+
+  // Save output
+  float pred[shape_out[2] * shape_out[3]];
+  unsigned char cbuf[shape_out[2] * shape_out[3]];
+
+  for (int i = 0; i < int(shape_out[2] * shape_out[3]); i++) {
+    pred[i] = static_cast<float>(outptr[i]);
+    cbuf[i] = static_cast<unsigned char>((outptr[i]) * 255);
+  }
+
+  cv::Mat cbuf_map(shape_out[2], shape_out[3], CV_8UC1,
+                   reinterpret_cast<unsigned char *>(cbuf));
+  cv::Mat pred_map(shape_out[2], shape_out[3], CV_32F,
+                   reinterpret_cast<float *>(pred));
+
+  const double threshold = double(Config["det_db_thresh"]) * 255;
+  const double maxvalue = 255;
+  cv::Mat bit_map;
+  cv::threshold(cbuf_map, bit_map, threshold, maxvalue, cv::THRESH_BINARY);
+
+  auto boxes = BoxesFromBitmap(pred_map, bit_map, Config);
+
+  std::vector<std::vector<std::vector<int>>> filter_boxes =
+      FilterTagDetRes(boxes, ratio_hw[0], ratio_hw[1], srcimg);
+
+  return filter_boxes;
+}
+
+std::shared_ptr<PaddlePredictor> loadModel(std::string model_file) {
+  MobileConfig config;
+  config.set_model_from_file(model_file);
+
+  std::shared_ptr<PaddlePredictor> predictor =
+      CreatePaddlePredictor<MobileConfig>(config);
+  return predictor;
+}
+
+cv::Mat Visualization(cv::Mat srcimg,
+                      std::vector<std::vector<std::vector<int>>> boxes) {
+  cv::Point rook_points[boxes.size()][4];
+  for (int n = 0; n < boxes.size(); n++) {
+    for (int m = 0; m < boxes[0].size(); m++) {
+      rook_points[n][m] = cv::Point(static_cast<int>(boxes[n][m][0]),
+                                    static_cast<int>(boxes[n][m][1]));
+    }
+  }
+  cv::Mat img_vis;
+  srcimg.copyTo(img_vis);
+  for (int n = 0; n < boxes.size(); n++) {
+    const cv::Point *ppt[1] = {rook_points[n]};
+    int npt[] = {4};
+    cv::polylines(img_vis, ppt, npt, 1, 1, CV_RGB(0, 255, 0), 2, 8, 0);
+  }
+
+  cv::imwrite("./vis.jpg", img_vis);
+  std::cout << "The detection visualized image saved in ./vis.jpg" << std::endl;
+  return img_vis;
+}
+
+std::vector<std::string> split(const std::string &str,
+                               const std::string &delim) {
+  std::vector<std::string> res;
+  if ("" == str)
+    return res;
+  char *strs = new char[str.length() + 1];
+  std::strcpy(strs, str.c_str());
+
+  char *d = new char[delim.length() + 1];
+  std::strcpy(d, delim.c_str());
+
+  char *p = std::strtok(strs, d);
+  while (p) {
+    string s = p;
+    res.push_back(s);
+    p = std::strtok(NULL, d);
+  }
+
+  return res;
+}
+
+std::map<std::string, double> LoadConfigTxt(std::string config_path) {
+  auto config = ReadDict(config_path);
+
+  std::map<std::string, double> dict;
+  for (int i = 0; i < config.size(); i++) {
+    std::vector<std::string> res = split(config[i], " ");
+    dict[res[0]] = stod(res[1]);
+  }
+  return dict;
+}
+
+int main(int argc, char **argv) {
+  if (argc < 5) {
+    std::cerr << "[ERROR] usage: " << argv[0]
+              << " det_model_file rec_model_file image_path\n";
+    exit(1);
+  }
+  std::string det_model_file = argv[1];
+  std::string rec_model_file = argv[2];
+  std::string img_path = argv[3];
+  std::string dict_path = argv[4];
+
+  //// load config from txt file
+  auto Config = LoadConfigTxt("./config.txt");
+
+  auto start = std::chrono::system_clock::now();
+
+  auto det_predictor = loadModel(det_model_file);
+  auto rec_predictor = loadModel(rec_model_file);
+
+  auto charactor_dict = ReadDict(dict_path);
+  charactor_dict.push_back(" ");
+
+  cv::Mat srcimg = cv::imread(img_path, cv::IMREAD_COLOR);
+  auto boxes = RunDetModel(det_predictor, srcimg, Config);
+
+  std::vector<std::string> rec_text;
+  std::vector<float> rec_text_score;
+  RunRecModel(boxes, srcimg, rec_predictor, rec_text, rec_text_score,
+              charactor_dict);
+
+  auto end = std::chrono::system_clock::now();
+  auto duration =
+      std::chrono::duration_cast<std::chrono::microseconds>(end - start);
+
+  //// visualization
+  auto img_vis = Visualization(srcimg, boxes);
+
+  //// print recognized text
+  for (int i = 0; i < rec_text.size(); i++) {
+    std::cout << i << "\t" << rec_text[i] << "\t" << rec_text_score[i]
+              << std::endl;
+  }
+
+  std::cout << "花费了"
+            << double(duration.count()) *
+                   std::chrono::microseconds::period::num /
+                   std::chrono::microseconds::period::den
+            << "秒" << std::endl;
+
+  return 0;
+}

deploy/lite/prepare.sh

+#!/bin/bash
+
+mkdir -p  $1/demo/cxx/ocr/debug/
+cp  ../../ppocr/utils/ppocr_keys_v1.txt  $1/demo/cxx/ocr/debug/
+cp -r  ./*   $1/demo/cxx/ocr/
+cp ./config.txt  $1/demo/cxx/ocr/debug/
+cp ../../doc/imgs/11.jpg  $1/demo/cxx/ocr/debug/
+
+echo "Prepare Done"

deploy/lite/readme.md

+# 端侧部署
+
+本教程将介绍基于[Paddle Lite](https://github.com/PaddlePaddle/Paddle-Lite) 在移动端部署PaddleOCR超轻量中文检测、识别模型的详细步骤。
+
+Paddle Lite是飞桨轻量化推理引擎，为手机、IOT端提供高效推理能力，并广泛整合跨平台硬件，为端侧部署及应用落地问题提供轻量化的部署方案。
+
+
+## 1. 准备环境
+
+### 运行准备
+- 电脑（编译Paddle Lite）
+- 安卓手机（armv7或armv8）
+
+### 1.1 准备交叉编译环境
+交叉编译环境用于编译 Paddle Lite 和 PaddleOCR 的C++ demo。
+支持多种开发环境，不同开发环境的编译流程请参考对应文档。
+
+1. [Docker](https://paddle-lite.readthedocs.io/zh/latest/user_guides/source_compile.html#docker)
+2. [Linux](https://paddle-lite.readthedocs.io/zh/latest/user_guides/source_compile.html#android)
+3. [MAC OS](https://paddle-lite.readthedocs.io/zh/latest/user_guides/source_compile.html#id13)
+4. [Windows](https://paddle-lite.readthedocs.io/zh/latest/demo_guides/x86.html#id4)
+
+### 1.2 准备预测库
+
+预测库有两种获取方式：
+- 1. 直接下载，预测库下载链接如下：
+      |平台|预测库下载链接|
+      |-|-|
+      |Android|[arm7](https://paddlelite-data.bj.bcebos.com/Release/2.6.1/Android/inference_lite_lib.android.armv7.gcc.c++_static.with_extra.CV_ON.tar.gz) / [arm8](https://paddlelite-data.bj.bcebos.com/Release/2.6.1/Android/inference_lite_lib.android.armv8.gcc.c++_static.with_extra.CV_ON.tar.gz)|
+      |IOS|[arm7](https://paddlelite-data.bj.bcebos.com/Release/2.6.1/iOS/inference_lite_lib.ios.armv7.with_extra.CV_ON.tar.gz) / [arm8](https://paddlelite-data.bj.bcebos.com/Release/2.6.1/iOS/inference_lite_lib.ios64.armv8.with_extra.CV_ON.tar.gz)|
+      |x86(Linux)|[预测库](https://paddlelite-data.bj.bcebos.com/Release/2.6.1/X86/Linux/inference_lite_lib.x86.linux.tar.gz)|
+
+      注：如果是从下Paddle-Lite[官网文档](https://paddle-lite.readthedocs.io/zh/latest/user_guides/release_lib.html#android-toolchain-gcc)下载的预测库，
+      注意选择`with_extra=ON，with_cv=ON`的下载链接。
+
+- 2. 编译Paddle-Lite得到预测库，Paddle-Lite的编译方式如下：
+```
+git clone https://github.com/PaddlePaddle/Paddle-Lite.git
+cd Paddle-Lite
+git checkout develop
+./lite/tools/build_android.sh  --arch=armv8  --with_cv=ON --with_extra=ON
+```
+
+注意：编译Paddle-Lite获得预测库时，需要打开`--with_cv=ON --with_extra=ON`两个选项，`--arch`表示`arm`版本，这里指定为armv8，
+更多编译命令
+介绍请参考[链接](https://paddle-lite.readthedocs.io/zh/latest/user_guides/Compile/Android.html#id2)。
+
+直接下载预测库并解压后，可以得到`inference_lite_lib.android.armv8/`文件夹，通过编译Paddle-Lite得到的预测库位于
+`Paddle-Lite/build.lite.android.armv8.gcc/inference_lite_lib.android.armv8/`文件夹下。
+预测库的文件目录如下：
+```
+inference_lite_lib.android.armv8/
+|-- cxx                                        C++ 预测库和头文件
+|   |-- include                                C++ 头文件
+|   |   |-- paddle_api.h
+|   |   |-- paddle_image_preprocess.h
+|   |   |-- paddle_lite_factory_helper.h
+|   |   |-- paddle_place.h
+|   |   |-- paddle_use_kernels.h
+|   |   |-- paddle_use_ops.h
+|   |   `-- paddle_use_passes.h
+|   `-- lib                                           C++预测库
+|       |-- libpaddle_api_light_bundled.a             C++静态库
+|       `-- libpaddle_light_api_shared.so             C++动态库
+|-- java                                     Java预测库
+|   |-- jar
+|   |   `-- PaddlePredictor.jar
+|   |-- so
+|   |   `-- libpaddle_lite_jni.so
+|   `-- src
+|-- demo                                     C++和Java示例代码
+|   |-- cxx                                  C++  预测库demo
+|   `-- java                                 Java 预测库demo
+```
+
+## 2 开始运行
+
+### 2.1 模型优化
+
+Paddle-Lite 提供了多种策略来自动优化原始的模型，其中包括量化、子图融合、混合调度、Kernel优选等方法，使用Paddle-lite的opt工具可以自动
+对inference模型进行优化，优化后的模型更轻量，模型运行速度更快。
+
+下述表格中提供了优化好的超轻量中文模型：
+
+|模型简介|检测模型|识别模型|Paddle-Lite版本|
+|-|-|-|-|
+|超轻量级中文OCR opt优化模型|[下载地址](https://paddleocr.bj.bcebos.com/deploy/lite/ch_det_mv3_db_opt.nb)|[下载地址](https://paddleocr.bj.bcebos.com/deploy/lite/ch_rec_mv3_crnn_opt.nb)|develop|
+
+如果直接使用上述表格中的模型进行部署，可略过下述步骤，直接阅读 [2.2节](#2.2与手机联调)。
+
+如果要部署的模型不在上述表格中，则需要按照如下步骤获得优化后的模型。
+
+模型优化需要Paddle-Lite的opt可执行文件，可以通过编译Paddle-Lite源码获得，编译步骤如下：
+```
+# 如果准备环境时已经clone了Paddle-Lite，则不用重新clone Paddle-Lite
+git clone https://github.com/PaddlePaddle/Paddle-Lite.git
+cd Paddle-Lite
+git checkout develop
+# 启动编译
+./lite/tools/build.sh build_optimize_tool
+```
+
+编译完成后，opt文件位于`build.opt/lite/api/`下，可通过如下方式查看opt的运行选项和使用方式；
+```
+cd build.opt/lite/api/
+./opt
+```
+
+|选项|说明|
+|-|-|
+|--model_dir|待优化的PaddlePaddle模型（非combined形式）的路径|
+|--model_file|待优化的PaddlePaddle模型（combined形式）的网络结构文件路径|
+|--param_file|待优化的PaddlePaddle模型（combined形式）的权重文件路径|
+|--optimize_out_type|输出模型类型，目前支持两种类型：protobuf和naive_buffer，其中naive_buffer是一种更轻量级的序列化/反序列化实现。若您需要在mobile端执行模型预测，请将此选项设置为naive_buffer。默认为protobuf|
+|--optimize_out|优化模型的输出路径|
+|--valid_targets|指定模型可执行的backend，默认为arm。目前可支持x86、arm、opencl、npu、xpu，可以同时指定多个backend(以空格分隔)，Model Optimize Tool将会自动选择最佳方式。如果需要支持华为NPU（Kirin 810/990 Soc搭载的达芬奇架构NPU），应当设置为npu, arm|
+|--record_tailoring_info|当使用 根据模型裁剪库文件 功能时，则设置该选项为true，以记录优化后模型含有的kernel和OP信息，默认为false|
+
+`--model_dir`适用于待优化的模型是非combined方式，PaddleOCR的inference模型是combined方式，即模型结构和模型参数使用单独一个文件存储。
+
+下面以PaddleOCR的超轻量中文模型为例，介绍使用编译好的opt文件完成inference模型到Paddle-Lite优化模型的转换。
+
+```
+# 下载PaddleOCR的超轻量文inference模型，并解压
+wget  https://paddleocr.bj.bcebos.com/ch_models/ch_det_mv3_db_infer.tar && tar xf ch_det_mv3_db_infer.tar
+wget  https://paddleocr.bj.bcebos.com/ch_models/ch_rec_mv3_crnn_infer.tar && tar xf ch_rec_mv3_crnn_infer.tar
+
+# 转换检测模型
+./opt --model_file=./ch_det_mv3_db/model --param_file=./ch_det_mv3_db/params --optimize_out_type=naive_buffer --optimize_out=./ch_det_mv3_db_opt --valid_targets=arm
+
+# 转换识别模型
+./opt --model_file=./ch_rec_mv3_crnn/model --param_file=./ch_rec_mv3_crnn/params --optimize_out_type=naive_buffer --optimize_out=./ch_rec_mv3_crnn_opt --valid_targets=arm
+```
+
+转换成功后，当前目录下会多出`ch_det_mv3_db_opt.nb`, `ch_rec_mv3_crnn_opt.nb`结尾的文件，即是转换成功的模型文件。
+
+注意：使用paddle-lite部署时，需要使用opt工具优化后的模型。 opt 转换的输入模型是paddle保存的inference模型
+
+<a name="2.2与手机联调"></a>
+### 2.2 与手机联调
+
+首先需要进行一些准备工作。
+ 1. 准备一台arm8的安卓手机，如果编译的预测库和opt文件是armv7，则需要arm7的手机，并修改Makefile中`ARM_ABI = arm7`。
+ 2. 打开手机的USB调试选项，选择文件传输模式，连接电脑。
+ 3. 电脑上安装adb工具，用于调试。 adb安装方式如下：
+
+    3.1. MAC电脑安装ADB:
+    ```
+    brew cask install android-platform-tools
+    ```
+    3.2. Linux安装ADB
+    ```
+    sudo apt update
+    sudo apt install -y wget adb
+    ```
+    3.3. Window安装ADB
+
+    win上安装需要去谷歌的安卓平台下载adb软件包进行安装：[链接](https://developer.android.com/studio)
+
+    打开终端，手机连接电脑，在终端中输入
+    ```
+    adb devices
+    ```
+    如果有device输出，则表示安装成功。
+    ```
+       List of devices attached
+       744be294    device
+    ```
+
+ 4. 准备优化后的模型、预测库文件、测试图像和使用的字典文件。
+ ```
+ git clone https://github.com/PaddlePaddle/PaddleOCR.git
+ cd PaddleOCR/deploy/lite/
+ # 运行prepare.sh，准备预测库文件、测试图像和使用的字典文件，并放置在预测库中的demo/cxx/ocr文件夹下
+ sh prepare.sh /{lite prediction library path}/inference_lite_lib.android.armv8
+
+ # 进入OCR demo的工作目录
+ cd /{lite prediction library path}/inference_lite_lib.android.armv8/
+ cd demo/cxx/ocr/
+ # 将C++预测动态库so文件复制到debug文件夹中
+ cp ../../../cxx/lib/libpaddle_light_api_shared.so ./debug/
+ ```
+
+ 准备测试图像，以`PaddleOCR/doc/imgs/11.jpg`为例，将测试的图像复制到`demo/cxx/ocr/debug/`文件夹下。
+ 准备lite opt工具优化后的模型文件，`ch_det_mv3_db_opt.nb，ch_rec_mv3_crnn_opt.nb`，放置在`demo/cxx/ocr/debug/`文件夹下。
+
+ 执行完成后，ocr文件夹下将有如下文件格式：
+
+```
+demo/cxx/ocr/
+|-- debug/  
+|   |--ch_det_mv3_db_opt.nb             优化后的检测模型文件
+|   |--ch_rec_mv3_crnn_opt.nb           优化后的识别模型文件
+|   |--11.jpg                           待测试图像
+|   |--ppocr_keys_v1.txt                字典文件
+|   |--libpaddle_light_api_shared.so    C++预测库文件
+|   |--config.txt                       DB-CRNN超参数配置
+|-- config.txt                  DB-CRNN超参数配置
+|-- crnn_process.cc             识别模型CRNN的预处理和后处理文件
+|-- crnn_process.h
+|-- db_post_process.cc          检测模型DB的后处理文件
+|-- db_post_process.h
+|-- Makefile                    编译文件
+|-- ocr_db_crnn.cc              C++预测源文件
+
+```
+
+ 5. 启动调试
+
+ 上述步骤完成后就可以使用adb将文件push到手机上运行，步骤如下：
+
+ ```
+ # 执行编译，得到可执行文件ocr_db_crnn
+ # ocr_db_crnn可执行文件的使用方式为:
+ # ./ocr_db_crnn  检测模型文件  识别模型文件  测试图像路径
+ make -j
+ # 将编译的可执行文件移动到debug文件夹中
+ mv ocr_db_crnn ./debug/
+ # 将debug文件夹push到手机上
+ adb push debug /data/local/tmp/
+ adb shell
+ cd /data/local/tmp/debug
+ export LD_LIBRARY_PATH=/data/local/tmp/debug:$LD_LIBRARY_PATH
+ ./ocr_db_crnn ch_det_mv3_db_opt.nb  ch_rec_mv3_crnn_opt.nb ./11.jpg  ppocr_keys_v1.txt
+ ```
+
+ 如果对代码做了修改，则需要重新编译并push到手机上。
+
+ 运行效果如下：
+
+<div align="center">
+    <img src="../imgs/demo.png" width="600">
+</div>

deploy/lite/readme_en.md

+
+# Tutorial of PaddleOCR Mobile deployment
+
+This tutorial will introduce how to use paddle-lite to deploy paddleOCR ultra-lightweight Chinese and English detection models on mobile phones.
+
+paddle-lite is a lightweight inference engine for PaddlePaddle.
+It provides efficient inference capabilities for mobile phones and IOTs,
+and extensively integrates cross-platform hardware to provide lightweight
+deployment solutions for end-side deployment issues.
+
+## 1. Preparation
+
+- Computer (for Compiling Paddle Lite)
+- Mobile phone (arm7 or arm8)
+
+## 2. Build ncnn library
+[build for Docker](https://paddle-lite.readthedocs.io/zh/latest/user_guides/source_compile.html#docker)
+[build for Linux](https://paddle-lite.readthedocs.io/zh/latest/user_guides/source_compile.html#android)
+[build for MAC OS](https://paddle-lite.readthedocs.io/zh/latest/user_guides/source_compile.html#id13)
+[build for windows](https://paddle-lite.readthedocs.io/zh/latest/demo_guides/x86.html#id4)
+
+## 3. Download prebuild library for android and ios
+
+|Platform|Prebuild library Download Link|
+|-|-|
+|Android|[arm7](https://paddlelite-data.bj.bcebos.com/Release/2.6.1/Android/inference_lite_lib.android.armv7.gcc.c++_static.with_extra.CV_ON.tar.gz) / [arm8](https://paddlelite-data.bj.bcebos.com/Release/2.6.1/Android/inference_lite_lib.android.armv8.gcc.c++_static.with_extra.CV_ON.tar.gz)|
+|IOS|[arm7](https://paddlelite-data.bj.bcebos.com/Release/2.6.1/iOS/inference_lite_lib.ios.armv7.with_extra.CV_ON.tar.gz) / [arm8](https://paddlelite-data.bj.bcebos.com/Release/2.6.1/iOS/inference_lite_lib.ios64.armv8.with_extra.CV_ON.tar.gz)|
+|x86(Linux)|[预测库](https://paddlelite-data.bj.bcebos.com/Release/2.6.1/X86/Linux/inference_lite_lib.x86.linux.tar.gz)|
+
+
+The structure of the prediction library is as follows:
+
+```
+inference_lite_lib.android.armv8/
+|-- cxx                                        C++ prebuild library
+|   |-- include                                C++
+|   |   |-- paddle_api.h
+|   |   |-- paddle_image_preprocess.h
+|   |   |-- paddle_lite_factory_helper.h
+|   |   |-- paddle_place.h
+|   |   |-- paddle_use_kernels.h
+|   |   |-- paddle_use_ops.h
+|   |   `-- paddle_use_passes.h
+|   `-- lib  
+|       |-- libpaddle_api_light_bundled.a             C++ static library
+|       `-- libpaddle_light_api_shared.so             C++ dynamic library
+|-- java                                     Java predict library
+|   |-- jar
+|   |   `-- PaddlePredictor.jar
+|   |-- so
+|   |   `-- libpaddle_lite_jni.so
+|   `-- src
+|-- demo                                     C++ and java demo
+|   |-- cxx  
+|   `-- java  
+```
+
+
+
+## 4. Inference Model Optimization
+
+Paddle Lite provides a variety of strategies to automatically optimize the original training model, including quantization, sub-graph fusion, hybrid scheduling, Kernel optimization and so on. In order to make the optimization process more convenient and easy to use, Paddle Lite provide opt tools to automatically complete the optimization steps and output a lightweight, optimal executable model.
+
+If you use PaddleOCR 8.6M OCR model to deploy, you can directly download the optimized model.
+
+
+|Introduction|Detection model|Recognition model|Paddle Lite branch |
+|-|-|-|-|
+|lightweight Chinese OCR optimized model|[Download](https://paddleocr.bj.bcebos.com/deploy/lite/ch_det_mv3_db_opt.nb)|[Download](https://paddleocr.bj.bcebos.com/deploy/lite/ch_rec_mv3_crnn_opt.nb)|develop|
+
+If the model to be deployed is not in the above table, you need to follow the steps below to obtain the optimized model.
+
+```
+git clone https://github.com/PaddlePaddle/Paddle-Lite.git
+cd Paddle-Lite
+git checkout develop
+./lite/tools/build.sh build_optimize_tool
+```
+
+The `opt` tool can be obtained by compiling Paddle Lite.
+
+After the compilation is complete, the opt file is located under `build.opt/lite/api/`.
+
+The `opt` can optimize the inference model saved by paddle.io.save_inference_model to get the model that the paddlelite API can use.
+
+The usage of opt is as follows：
+```
+wget  https://paddleocr.bj.bcebos.com/ch_models/ch_det_mv3_db_infer.tar && tar xf ch_det_mv3_db_infer.tar
+wget  https://paddleocr.bj.bcebos.com/ch_models/ch_rec_mv3_crnn_infer.tar && tar xf ch_rec_mv3_crnn_infer.tar
+
+./opt --model_file=./ch_det_mv3_db/model --param_file=./ch_det_mv3_db/params --optimize_out_type=naive_buffer --optimize_out=./ch_det_mv3_db_opt --valid_targets=arm
+./opt --model_file=./ch_rec_mv3_crnn/model --param_file=./ch_rec_mv3_crnn/params --optimize_out_type=naive_buffer --optimize_out=./ch_rec_mv3_crnn_opt --valid_targets=arm
+
+```
+
+When the above code command is completed, there will be two more files `ch_det_mv3_db_opt.nb`,
+`ch_rec_mv3_crnn_opt.nb` in the current directory, which is the converted model file.
+
+## 5. Run optimized model on Phone
+
+1. Prepare an Android phone with arm8. If the compiled prediction library and opt file are armv7, you need an arm7 phone and modify ARM_ABI = arm7 in the Makefile.
+
+2. Make sure the phone is connected to the computer, open the USB debugging option of the phone, and select the file transfer mode.
+
+3. Install the adb tool on the computer.
+    3.1 Install ADB for MAC
+    ```
+    brew cask install android-platform-tools
+    ```
+    3.2 Install ADB for Linux
+    ```
+    sudo apt update
+    sudo apt install -y wget adb
+    ```
+    3.3 Install ADB for windows
+    [Download Link](https://developer.android.com/studio)
+
+    Verify whether adb is installed successfully
+    ```
+    $ adb devices
+
+    List of devices attached
+    744be294    device
+    ```
+
+    If there is `device` output, it means the installation was successful.
+
+4. Prepare optimized models, prediction library files, test images and dictionary files used.
+
+```
+ git clone https://github.com/PaddlePaddle/PaddleOCR.git
+ cd PaddleOCR/deploy/lite/
+ # run prepare.sh
+ sh prepare.sh /{lite prediction library path}/inference_lite_lib.android.armv8
+
+ #
+ cd /{lite prediction library path}/inference_lite_lib.android.armv8/
+ cd demo/cxx/ocr/
+ # copy paddle-lite C++ .so file to debug/ directory
+ cp ../../../cxx/lib/libpaddle_light_api_shared.so ./debug/
+
+ cd inference_lite_lib.android.armv8/demo/cxx/ocr/
+ cp ../../../cxx/lib/libpaddle_light_api_shared.so ./debug/
+
+```
+
+Prepare the test image, taking `PaddleOCR/doc/imgs/11.jpg` as an example, copy the image file to the `demo/cxx/ocr/debug/` folder.
+Prepare the model files optimized by the lite opt tool, `ch_det_mv3_db_opt.nb, ch_rec_mv3_crnn_opt.nb`,
+and place them under the `demo/cxx/ocr/debug/` folder.
+
+
+The structure of the OCR demo is as follows after the above command is executed:
+```
+demo/cxx/ocr/
+|-- debug/  
+|   |--ch_det_mv3_db_opt.nb             Detection model
+|   |--ch_rec_mv3_crnn_opt.nb           Recognition model
+|   |--11.jpg                           Image for OCR
+|   |--ppocr_keys_v1.txt                Dictionary file
+|   |--libpaddle_light_api_shared.so    C++ .so file
+|   |--config.txt                       Config file
+|-- config.txt  
+|-- crnn_process.cc  
+|-- crnn_process.h
+|-- db_post_process.cc  
+|-- db_post_process.h
+|-- Makefile  
+|-- ocr_db_crnn.cc  
+
+```
+
+5. Run Model on phone
+
+```
+cd inference_lite_lib.android.armv8/demo/cxx/ocr/
+make -j
+mv ocr_db_crnn ./debug/
+adb push debug /data/local/tmp/
+adb shell
+cd /data/local/tmp/debug
+export LD_LIBRARY_PATH=/data/local/tmp/debug:$LD_LIBRARY_PATH
+# run model
+./ocr_db_crnn ch_det_mv3_db_opt.nb  ch_rec_mv3_crnn_opt.nb ./11.jpg  ppocr_keys_v1.txt
+```
+
+The outputs are as follows:
+
+<div align="center">
+    <img src="../imgs/demo.png" width="600">
+</div>

deploy/pdserving/det_local_server.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from paddle_serving_client import Client
+import cv2
+import sys
+import numpy as np
+import os
+from paddle_serving_client import Client
+from paddle_serving_app.reader import Sequential, ResizeByFactor
+from paddle_serving_app.reader import Div, Normalize, Transpose
+from paddle_serving_app.reader import DBPostProcess, FilterBoxes
+if sys.argv[1] == 'gpu':
+    from paddle_serving_server_gpu.web_service import WebService
+elif sys.argv[1] == 'cpu'
+    from paddle_serving_server.web_service import WebService
+import time
+import re
+import base64
+
+
+class OCRService(WebService):
+    def init_det(self):
+        self.det_preprocess = Sequential([
+            ResizeByFactor(32, 960), Div(255),
+            Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), Transpose(
+                (2, 0, 1))
+        ])
+        self.filter_func = FilterBoxes(10, 10)
+        self.post_func = DBPostProcess({
+            "thresh": 0.3,
+            "box_thresh": 0.5,
+            "max_candidates": 1000,
+            "unclip_ratio": 1.5,
+            "min_size": 3
+        })
+
+    def preprocess(self, feed=[], fetch=[]):
+        data = base64.b64decode(feed[0]["image"].encode('utf8'))
+        data = np.fromstring(data, np.uint8)
+        im = cv2.imdecode(data, cv2.IMREAD_COLOR)
+        self.ori_h, self.ori_w, _ = im.shape
+        det_img = self.det_preprocess(im)
+        _, self.new_h, self.new_w = det_img.shape
+        return {"image": det_img[np.newaxis, :].copy()}, ["concat_1.tmp_0"]
+
+    def postprocess(self, feed={}, fetch=[], fetch_map=None):
+        det_out = fetch_map["concat_1.tmp_0"]
+        ratio_list = [
+            float(self.new_h) / self.ori_h, float(self.new_w) / self.ori_w
+        ]
+        dt_boxes_list = self.post_func(det_out, [ratio_list])
+        dt_boxes = self.filter_func(dt_boxes_list[0], [self.ori_h, self.ori_w])
+        return {"dt_boxes": dt_boxes.tolist()}
+
+
+ocr_service = OCRService(name="ocr")
+ocr_service.load_model_config("ocr_det_model")
+if sys.argv[1] == 'gpu':
+    ocr_service.set_gpus("0")
+    ocr_service.prepare_server(workdir="workdir", port=9292, device="gpu", gpuid=0)
+elif sys.argv[1] == 'cpu':
+    ocr_service.prepare_server(workdir="workdir", port=9292)
+ocr_service.init_det()
+ocr_service.run_debugger_service()
+ocr_service.run_web_service()

deploy/pdserving/det_web_server.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from paddle_serving_client import Client
+import cv2
+import sys
+import numpy as np
+import os
+from paddle_serving_client import Client
+from paddle_serving_app.reader import Sequential, ResizeByFactor
+from paddle_serving_app.reader import Div, Normalize, Transpose
+from paddle_serving_app.reader import DBPostProcess, FilterBoxes
+if sys.argv[1] == 'gpu':
+    from paddle_serving_server_gpu.web_service import WebService
+elif sys.argv[1] == 'cpu':
+    from paddle_serving_server.web_service import WebService
+import time
+import re
+import base64
+
+
+class OCRService(WebService):
+    def init_det(self):
+        self.det_preprocess = Sequential([
+            ResizeByFactor(32, 960), Div(255),
+            Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), Transpose(
+                (2, 0, 1))
+        ])
+        self.filter_func = FilterBoxes(10, 10)
+        self.post_func = DBPostProcess({
+            "thresh": 0.3,
+            "box_thresh": 0.5,
+            "max_candidates": 1000,
+            "unclip_ratio": 1.5,
+            "min_size": 3
+        })
+
+    def preprocess(self, feed=[], fetch=[]):
+        data = base64.b64decode(feed[0]["image"].encode('utf8'))
+        data = np.fromstring(data, np.uint8)
+        im = cv2.imdecode(data, cv2.IMREAD_COLOR)
+        self.ori_h, self.ori_w, _ = im.shape
+        det_img = self.det_preprocess(im)
+        _, self.new_h, self.new_w = det_img.shape
+        print(det_img)
+        return {"image": det_img}, ["concat_1.tmp_0"]
+
+    def postprocess(self, feed={}, fetch=[], fetch_map=None):
+        det_out = fetch_map["concat_1.tmp_0"]
+        ratio_list = [
+            float(self.new_h) / self.ori_h, float(self.new_w) / self.ori_w
+        ]
+        dt_boxes_list = self.post_func(det_out, [ratio_list])
+        dt_boxes = self.filter_func(dt_boxes_list[0], [self.ori_h, self.ori_w])
+        return {"dt_boxes": dt_boxes.tolist()}
+
+
+ocr_service = OCRService(name="ocr")
+ocr_service.load_model_config("ocr_det_model")
+if sys.argv[1] == 'gpu':
+    ocr_service.set_gpus("0")
+    ocr_service.prepare_server(workdir="workdir", port=9292, device="gpu", gpuid=0)
+elif sys.argv[1] == 'cpu':
+    ocr_service.prepare_server(workdir="workdir", port=9292, device="cpu")
+ocr_service.init_det()
+ocr_service.run_rpc_service()
+ocr_service.run_web_service()

deploy/pdserving/ocr_local_server.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from paddle_serving_client import Client
+from paddle_serving_app.reader import OCRReader
+import cv2
+import sys
+import numpy as np
+import os
+from paddle_serving_client import Client
+from paddle_serving_app.reader import Sequential, URL2Image, ResizeByFactor
+from paddle_serving_app.reader import Div, Normalize, Transpose
+from paddle_serving_app.reader import DBPostProcess, FilterBoxes, GetRotateCropImage, SortedBoxes
+if sys.argv[1] == 'gpu':
+    from paddle_serving_server_gpu.web_service import WebService
+elif sys.argv[1] == 'cpu':
+    from paddle_serving_server.web_service import WebService
+from paddle_serving_app.local_predict import Debugger
+import time
+import re
+import base64
+
+
+class OCRService(WebService):
+    def init_det_debugger(self, det_model_config):
+        self.det_preprocess = Sequential([
+            ResizeByFactor(32, 960), Div(255),
+            Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), Transpose(
+                (2, 0, 1))
+        ])
+        self.det_client = Debugger()
+        if sys.argv[1] == 'gpu':
+            self.det_client.load_model_config(
+                det_model_config, gpu=True, profile=False)
+        elif sys.argv[1] == 'cpu':
+            self.det_client.load_model_config(
+                det_model_config, gpu=False, profile=False)
+        self.ocr_reader = OCRReader()
+
+    def preprocess(self, feed=[], fetch=[]):
+        data = base64.b64decode(feed[0]["image"].encode('utf8'))
+        data = np.fromstring(data, np.uint8)
+        im = cv2.imdecode(data, cv2.IMREAD_COLOR)
+        ori_h, ori_w, _ = im.shape
+        det_img = self.det_preprocess(im)
+        _, new_h, new_w = det_img.shape
+        det_img = det_img[np.newaxis, :]
+        det_img = det_img.copy()
+        det_out = self.det_client.predict(
+            feed={"image": det_img}, fetch=["concat_1.tmp_0"])
+        filter_func = FilterBoxes(10, 10)
+        post_func = DBPostProcess({
+            "thresh": 0.3,
+            "box_thresh": 0.5,
+            "max_candidates": 1000,
+            "unclip_ratio": 1.5,
+            "min_size": 3
+        })
+        sorted_boxes = SortedBoxes()
+        ratio_list = [float(new_h) / ori_h, float(new_w) / ori_w]
+        dt_boxes_list = post_func(det_out["concat_1.tmp_0"], [ratio_list])
+        dt_boxes = filter_func(dt_boxes_list[0], [ori_h, ori_w])
+        dt_boxes = sorted_boxes(dt_boxes)
+        get_rotate_crop_image = GetRotateCropImage()
+        img_list = []
+        max_wh_ratio = 0
+        for i, dtbox in enumerate(dt_boxes):
+            boximg = get_rotate_crop_image(im, dt_boxes[i])
+            img_list.append(boximg)
+            h, w = boximg.shape[0:2]
+            wh_ratio = w * 1.0 / h
+            max_wh_ratio = max(max_wh_ratio, wh_ratio)
+        if len(img_list) == 0:
+            return [], []
+        _, w, h = self.ocr_reader.resize_norm_img(img_list[0],
+                                                  max_wh_ratio).shape
+        imgs = np.zeros((len(img_list), 3, w, h)).astype('float32')
+        for id, img in enumerate(img_list):
+            norm_img = self.ocr_reader.resize_norm_img(img, max_wh_ratio)
+            imgs[id] = norm_img
+        feed = {"image": imgs.copy()}
+        fetch = ["ctc_greedy_decoder_0.tmp_0", "softmax_0.tmp_0"]
+        return feed, fetch
+
+    def postprocess(self, feed={}, fetch=[], fetch_map=None):
+        rec_res = self.ocr_reader.postprocess(fetch_map, with_score=True)
+        res_lst = []
+        for res in rec_res:
+            res_lst.append(res[0])
+        res = {"res": res_lst}
+        return res
+
+
+ocr_service = OCRService(name="ocr")
+ocr_service.load_model_config("ocr_rec_model")
+ocr_service.prepare_server(workdir="workdir", port=9292)
+ocr_service.init_det_debugger(det_model_config="ocr_det_model")
+if sys.argv[1] == 'gpu':
+    ocr_service.run_debugger_service(gpu=True)
+elif sys.argv[1] == 'cpu':
+    ocr_service.run_debugger_service()
+ocr_service.run_web_service()

deploy/pdserving/ocr_web_client.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# -*- coding: utf-8 -*-
+
+import requests
+import json
+import cv2
+import base64
+import os, sys
+import time
+
+def cv2_to_base64(image):
+    #data = cv2.imencode('.jpg', image)[1]
+    return base64.b64encode(image).decode(
+        'utf8')  #data.tostring()).decode('utf8')
+
+headers = {"Content-type": "application/json"}
+url = "http://127.0.0.1:9292/ocr/prediction"
+test_img_dir = "../../doc/imgs/"
+for img_file in os.listdir(test_img_dir):
+    with open(os.path.join(test_img_dir, img_file), 'rb') as file:
+        image_data1 = file.read()
+    image = cv2_to_base64(image_data1)
+    data = {"feed": [{"image": image}], "fetch": ["res"]}
+    r = requests.post(url=url, headers=headers, data=json.dumps(data))
+    print(r.json())

deploy/pdserving/ocr_web_server.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from paddle_serving_client import Client
+from paddle_serving_app.reader import OCRReader
+import cv2
+import sys
+import numpy as np
+import os
+from paddle_serving_client import Client
+from paddle_serving_app.reader import Sequential, URL2Image, ResizeByFactor
+from paddle_serving_app.reader import Div, Normalize, Transpose
+from paddle_serving_app.reader import DBPostProcess, FilterBoxes, GetRotateCropImage, SortedBoxes
+if sys.argv[1] == 'gpu':
+    from paddle_serving_server_gpu.web_service import WebService
+elif sys.argv[1] == 'cpu':
+    from paddle_serving_server.web_service import WebService
+import time
+import re
+import base64
+
+
+class OCRService(WebService):
+    def init_det_client(self, det_port, det_client_config):
+        self.det_preprocess = Sequential([
+            ResizeByFactor(32, 960), Div(255),
+            Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]), Transpose(
+                (2, 0, 1))
+        ])
+        self.det_client = Client()
+        self.det_client.load_client_config(det_client_config)
+        self.det_client.connect(["127.0.0.1:{}".format(det_port)])
+        self.ocr_reader = OCRReader()
+
+    def preprocess(self, feed=[], fetch=[]):
+        data = base64.b64decode(feed[0]["image"].encode('utf8'))
+        data = np.fromstring(data, np.uint8)
+        im = cv2.imdecode(data, cv2.IMREAD_COLOR)
+        ori_h, ori_w, _ = im.shape
+        det_img = self.det_preprocess(im)
+        det_out = self.det_client.predict(
+            feed={"image": det_img}, fetch=["concat_1.tmp_0"])
+        _, new_h, new_w = det_img.shape
+        filter_func = FilterBoxes(10, 10)
+        post_func = DBPostProcess({
+            "thresh": 0.3,
+            "box_thresh": 0.5,
+            "max_candidates": 1000,
+            "unclip_ratio": 1.5,
+            "min_size": 3
+        })
+        sorted_boxes = SortedBoxes()
+        ratio_list = [float(new_h) / ori_h, float(new_w) / ori_w]
+        dt_boxes_list = post_func(det_out["concat_1.tmp_0"], [ratio_list])
+        dt_boxes = filter_func(dt_boxes_list[0], [ori_h, ori_w])
+        dt_boxes = sorted_boxes(dt_boxes)
+        get_rotate_crop_image = GetRotateCropImage()
+        feed_list = []
+        img_list = []
+        max_wh_ratio = 0
+        for i, dtbox in enumerate(dt_boxes):
+            boximg = get_rotate_crop_image(im, dt_boxes[i])
+            img_list.append(boximg)
+            h, w = boximg.shape[0:2]
+            wh_ratio = w * 1.0 / h
+            max_wh_ratio = max(max_wh_ratio, wh_ratio)
+        for img in img_list:
+            norm_img = self.ocr_reader.resize_norm_img(img, max_wh_ratio)
+            feed = {"image": norm_img}
+            feed_list.append(feed)
+        fetch = ["ctc_greedy_decoder_0.tmp_0", "softmax_0.tmp_0"]
+        return feed_list, fetch
+
+    def postprocess(self, feed={}, fetch=[], fetch_map=None):
+        rec_res = self.ocr_reader.postprocess(fetch_map, with_score=True)
+        res_lst = []
+        for res in rec_res:
+            res_lst.append(res[0])
+        res = {"res": res_lst}
+        return res
+
+
+ocr_service = OCRService(name="ocr")
+ocr_service.load_model_config("ocr_rec_model")
+if sys.argv[1] == 'gpu':
+    ocr_service.set_gpus("0")
+    ocr_service.prepare_server(workdir="workdir", port=9292, device="gpu", gpuid=0)
+elif sys.argv[1] == 'cpu':
+    ocr_service.prepare_server(workdir="workdir", port=9292)
+ocr_service.init_det_client(
+    det_port=9293,
+    det_client_config="ocr_det_client/serving_client_conf.prototxt")
+ocr_service.run_rpc_service()
+ocr_service.run_web_service()

deploy/pdserving/readme.md

+# Paddle Serving 服务部署(Beta)
+
+本教程将介绍基于[Paddle Serving](https://github.com/PaddlePaddle/Serving)部署PaddleOCR在线预测服务的详细步骤。
+
+## 快速启动服务
+
+### 1. 准备环境
+我们先安装Paddle Serving相关组件
+我们推荐用户使用GPU来做Paddle Serving的OCR服务部署 
+
+**CUDA版本：9.0**
+
+**CUDNN版本：7.0**
+
+**操作系统版本：CentOS 6以上**
+
+**Python3操作指南：**
+```
+#以下提供beta版本的paddle serving whl包，欢迎试用，正式版会在8月中正式上线
+#GPU用户下载server包使用这个链接
+wget --no-check-certificate https://paddle-serving.bj.bcebos.com/others/paddle_serving_server_gpu-0.3.2-py3-none-any.whl
+python -m pip install paddle_serving_server_gpu-0.3.2-py3-none-any.whl
+#CPU版本使用这个链接
+wget --no-check-certificate https://paddle-serving.bj.bcebos.com/others/paddle_serving_server-0.3.2-py3-none-any.whl
+python -m pip install paddle_serving_server-0.3.2-py3-none-any.whl
+#客户端和App包使用以下链接（CPU，GPU通用）
+wget --no-check-certificate https://paddle-serving.bj.bcebos.com/others/paddle_serving_client-0.3.2-cp36-none-any.whl
+wget --no-check-certificate https://paddle-serving.bj.bcebos.com/others/paddle_serving_app-0.1.2-py3-none-any.whl
+python -m pip install paddle_serving_app-0.1.2-py3-none-any.whl paddle_serving_client-0.3.2-cp36-none-any.whl
+```
+
+**Python2操作指南：**
+```
+#以下提供beta版本的paddle serving whl包，欢迎试用，正式版会在8月中正式上线
+#GPU用户下载server包使用这个链接
+wget --no-check-certificate https://paddle-serving.bj.bcebos.com/others/paddle_serving_server_gpu-0.3.2-py2-none-any.whl
+python -m pip install paddle_serving_server_gpu-0.3.2-py2-none-any.whl 
+#CPU版本使用这个链接
+wget --no-check-certificate https://paddle-serving.bj.bcebos.com/others/paddle_serving_server-0.3.2-py2-none-any.whl
+python -m pip install paddle_serving_server-0.3.2-py2-none-any.whl
+
+#客户端和App包使用以下链接（CPU，GPU通用）
+wget --no-check-certificate https://paddle-serving.bj.bcebos.com/others/paddle_serving_app-0.1.2-py2-none-any.whl
+wget --no-check-certificate https://paddle-serving.bj.bcebos.com/others/paddle_serving_client-0.3.2-cp27-none-any.whl
+python -m pip install paddle_serving_app-0.1.2-py2-none-any.whl paddle_serving_client-0.3.2-cp27-none-any.whl
+```
+
+### 2. 模型转换
+可以使用`paddle_serving_app`提供的模型，执行下列命令
+```
+python -m paddle_serving_app.package --get_model ocr_rec
+tar -xzvf ocr_rec.tar.gz
+python -m paddle_serving_app.package --get_model ocr_det
+tar -xzvf ocr_det.tar.gz 
+```
+执行上述命令会下载`db_crnn_mobile`的模型，如果想要下载规模更大的`db_crnn_server`模型，可以在下载预测模型并解压之后。参考[如何从Paddle保存的预测模型转为Paddle Serving格式可部署的模型](https://github.com/PaddlePaddle/Serving/blob/develop/doc/INFERENCE_TO_SERVING_CN.md)。
+
+### 3. 启动服务
+启动服务可以根据实际需求选择启动`标准版`或者`快速版`，两种方式的对比如下表：  
+
+|版本|特点|适用场景|
+|-|-|-|
+|标准版|稳定性高，分布式部署|适用于吞吐量大，需要跨机房部署的情况|
+|快速版|部署方便，预测速度快|适用于对预测速度要求高，迭代速度快的场景|
+
+#### 方式1. 启动标准版服务
+
+```
+# cpu，gpu启动二选一，以下是cpu启动
+python -m paddle_serving_server.serve --model ocr_det_model --port 9293 
+python ocr_web_server.py cpu
+# gpu启动
+python -m paddle_serving_server_gpu.serve --model ocr_det_model --port 9293 --gpu_id 0
+python ocr_web_server.py gpu
+```
+
+#### 方式2. 启动快速版服务
+
+```
+# cpu，gpu启动二选一，以下是cpu启动
+python ocr_local_server.py cpu
+# gpu启动
+python ocr_local_server.py gpu
+```
+
+## 发送预测请求
+
+```
+python ocr_web_client.py
+```
+
+## 返回结果格式说明
+
+返回结果是json格式
+```
+{u'result': {u'res': [u'\u571f\u5730\u6574\u6cbb\u4e0e\u571f\u58e4\u4fee\u590d\u7814\u7a76\u4e2d\u5fc3', u'\u534e\u5357\u519c\u4e1a\u5927\u5b661\u7d20\u56fe']}}
+```
+我们也可以打印结果json串中`res`字段的每一句话
+```
+土地整治与土壤修复研究中心
+华南农业大学1素图
+```
+
+## 自定义修改服务逻辑
+
+在`ocr_web_server.py`或是`ocr_local_server.py`当中的`preprocess`函数里面做了检测服务和识别服务的前处理，`postprocess`函数里面做了识别的后处理服务，可以在相应的函数中做修改。调用了`paddle_serving_app`库提供的常见CV模型的前处理/后处理库。
+
+如果想要单独启动Paddle Serving的检测服务和识别服务，参见下列表格, 执行对应的脚本即可，并且在命令行参数注明用的CPU或是GPU来提供服务。
+
+| 模型 | 标准版         | 快速版           |
+| ---- | ----------------- | ------------------- |
+| 检测 | det_web_server.py | det_local_server.py |
+| 识别 | rec_web_server.py | rec_local_server.py |
+
+更多信息参见[Paddle Serving](https://github.com/PaddlePaddle/Serving)

deploy/pdserving/rec_local_server.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from paddle_serving_client import Client
+from paddle_serving_app.reader import OCRReader
+import cv2
+import sys
+import numpy as np
+import os
+from paddle_serving_client import Client
+from paddle_serving_app.reader import Sequential, URL2Image, ResizeByFactor
+from paddle_serving_app.reader import Div, Normalize, Transpose
+from paddle_serving_app.reader import DBPostProcess, FilterBoxes, GetRotateCropImage, SortedBoxes
+from paddle_serving_server_gpu.web_service import WebService
+import time
+import re
+import base64
+
+
+class OCRService(WebService):
+    def init_rec(self):
+        self.ocr_reader = OCRReader()
+
+    def preprocess(self, feed=[], fetch=[]):
+        img_list = []
+        for feed_data in feed:
+            data = base64.b64decode(feed_data["image"].encode('utf8'))
+            data = np.fromstring(data, np.uint8)
+            im = cv2.imdecode(data, cv2.IMREAD_COLOR)
+            img_list.append(im)
+        max_wh_ratio = 0
+        for i, boximg in enumerate(img_list):
+            h, w = boximg.shape[0:2]
+            wh_ratio = w * 1.0 / h
+            max_wh_ratio = max(max_wh_ratio, wh_ratio)
+        _, w, h = self.ocr_reader.resize_norm_img(img_list[0],
+                                                  max_wh_ratio).shape
+        imgs = np.zeros((len(img_list), 3, w, h)).astype('float32')
+        for i, img in enumerate(img_list):
+            norm_img = self.ocr_reader.resize_norm_img(img, max_wh_ratio)
+            imgs[i] = norm_img
+        feed = {"image": imgs.copy()}
+        fetch = ["ctc_greedy_decoder_0.tmp_0", "softmax_0.tmp_0"]
+        return feed, fetch
+
+    def postprocess(self, feed={}, fetch=[], fetch_map=None):
+        rec_res = self.ocr_reader.postprocess(fetch_map, with_score=True)
+        res_lst = []
+        for res in rec_res:
+            res_lst.append(res[0])
+        res = {"res": res_lst}
+        return res
+
+
+ocr_service = OCRService(name="ocr")
+ocr_service.load_model_config("ocr_rec_model")
+ocr_service.set_gpus("0")
+ocr_service.init_rec()
+ocr_service.prepare_server(workdir="workdir", port=9292, device="gpu", gpuid=0)
+ocr_service.run_debugger_service()
+ocr_service.run_web_service()

deploy/pdserving/rec_web_server.py

+# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from paddle_serving_client import Client
+from paddle_serving_app.reader import OCRReader
+import cv2
+import sys
+import numpy as np
+import os
+from paddle_serving_client import Client
+from paddle_serving_app.reader import Sequential, URL2Image, ResizeByFactor
+from paddle_serving_app.reader import Div, Normalize, Transpose
+from paddle_serving_app.reader import DBPostProcess, FilterBoxes, GetRotateCropImage, SortedBoxes
+if sys.argv[1] == 'gpu':
+    from paddle_serving_server_gpu.web_service import WebService
+elif sys.argv[1] == 'cpu':
+    from paddle_serving_server.web_service import WebService
+import time
+import re
+import base64
+
+
+class OCRService(WebService):
+    def init_rec(self):
+        self.ocr_reader = OCRReader()
+
+    def preprocess(self, feed=[], fetch=[]):
+        # TODO: to handle batch rec images
+        img_list = []
+        for feed_data in feed:
+            data = base64.b64decode(feed_data["image"].encode('utf8'))
+            data = np.fromstring(data, np.uint8)
+            im = cv2.imdecode(data, cv2.IMREAD_COLOR)
+            img_list.append(im)
+        feed_list = []
+        max_wh_ratio = 0
+        for i, boximg in enumerate(img_list):
+            h, w = boximg.shape[0:2]
+            wh_ratio = w * 1.0 / h
+            max_wh_ratio = max(max_wh_ratio, wh_ratio)
+        for img in img_list:
+            norm_img = self.ocr_reader.resize_norm_img(img, max_wh_ratio)
+            feed = {"image": norm_img}
+            feed_list.append(feed)
+        fetch = ["ctc_greedy_decoder_0.tmp_0", "softmax_0.tmp_0"]
+        return feed_list, fetch
+
+    def postprocess(self, feed={}, fetch=[], fetch_map=None):
+        rec_res = self.ocr_reader.postprocess(fetch_map, with_score=True)
+        res_lst = []
+        for res in rec_res:
+            res_lst.append(res[0])
+        res = {"res": res_lst}
+        return res
+
+
+ocr_service = OCRService(name="ocr")
+ocr_service.load_model_config("ocr_rec_model")
+ocr_service.init_rec()
+if sys.argv[1] == 'gpu':
+    ocr_service.set_gpus("0")
+    ocr_service.prepare_server(workdir="workdir", port=9292, device="gpu", gpuid=0)
+elif sys.argv[1] == 'cpu':
+    ocr_service.prepare_server(workdir="workdir", port=9292, device="cpu")
+ocr_service.run_rpc_service()
+ocr_service.run_web_service()