Merge branch 'dygraph' of https://github.com/PaddlePaddle/PaddleOCR into fix_vqa

test_tipc/prepare.sh

@@ -179,7 +179,7 @@ elif [ ${MODE} = "whole_infer" ];then
         cd ./inference/ && tar xf rec_r34_vd_tps_bilstm_ctc_v2.0_train.tar && cd ../
     fi
     if [ ${model_name} == "ch_ppocr_server_v2.0_rec" ]; then
-        wget -nc -P ./inference/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/en/ch_ppocr_server_v2.0_rec_train.tar --no-check-certificate
+        wget -nc -P ./inference/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_server_v2.0_rec_train.tar --no-check-certificate
         cd ./inference/ && tar xf ch_ppocr_server_v2.0_rec_train.tar && cd ../
     fi
     if [ ${model_name} == "ch_ppocr_mobile_v2.0_rec" ]; then
@@ -239,18 +239,21 @@ fi
 
 if [ ${MODE} = "klquant_whole_infer" ]; then
     wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/icdar2015_lite.tar --no-check-certificate
-    cd ./train_data/ && tar xf icdar2015_lite.tar
-    ln -s ./icdar2015_lite ./icdar2015 && cd ../
+    cd ./train_data/ && tar xf icdar2015_lite.tar && rm -rf ./icdar2015 && ln -s ./icdar2015_lite ./icdar2015 && cd ../
     if [ ${model_name} = "ch_ppocr_mobile_v2.0_det_KL" ]; then
         wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_det_infer.tar --no-check-certificate
         wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar  --no-check-certificate
         cd ./inference && tar xf ch_ppocr_mobile_v2.0_det_infer.tar && tar xf ch_det_data_50.tar && cd ../ 
     fi
-    if [ ${model_name} = "ch_PPOCRv2_det" ]; then
-        eval_model_name="ch_PP-OCRv2_det_infer"
+    if [ ${model_name} = "PPOCRv2_ocr_rec_kl" ]; then
+        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_rec_infer.tar  --no-check-certificate
+        wget -nc -P ./inference/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/rec_inference.tar  --no-check-certificate
+        cd ./inference && tar xf rec_inference.tar && tar xf ch_PP-OCRv2_rec_infer.tar && cd ../
+    fi
+    if [ ${model_name} = "PPOCRv2_ocr_det_kl" ]; then
         wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar  --no-check-certificate
-        wget -nc -P ./inference/ https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_det_infer.tar  --no-check-certificate
-        cd ./inference && tar xf ${eval_model_name}.tar && tar xf ch_det_data_50.tar && cd ../
+        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_det_infer.tar  --no-check-certificate
+        cd ./inference && tar xf ch_PP-OCRv2_det_infer.tar && tar xf ch_det_data_50.tar && cd ../
     fi 
     if [ ${model_name} = "ch_ppocr_mobile_v2.0_rec_KL" ]; then
         wget -nc -P ./inference/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_rec_infer.tar --no-check-certificate

test_tipc/prepare_js.sh

+#!/bin/bash
+
+set -o errexit
+set -o nounset
+shopt -s extglob
+
+# paddlejs prepare 主要流程
+# 1. 判断 node, npm 是否安装
+# 2. 下载测试模型，当前检测模型是 ch_PP-OCRv2_det_infer ，识别模型是 ch_PP-OCRv2_rec_infer [1, 3, 32, 320]。如果需要替换模型，可直接将模型文件放在test_tipc/web/models/目录下。
+#  - 文本检测模型：https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_det_infer.tar
+#  - 文本识别模型：https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_rec_infer.tar
+#  - 文本识别模型[1, 3, 32, 320]：https://paddlejs.bj.bcebos.com/models/ch_PP-OCRv2_rec_infer.tar
+#  - 保证较为准确的识别效果，需要将文本识别模型导出为输入shape[1, 3, 32, 320]的静态模型
+# 3. 转换模型， model.pdmodel model.pdiparams 转换为 model.json chunk.dat（检测模型保存地址：test_tipc/web/models/ch_PP-OCRv2/det，识别模型保存地址：test_tipc/web/models/ch_PP-OCRv2/rec）
+# 4. 安装最新版本 ocr sdk  @paddlejs-models/ocr@latest
+# 5. 安装测试环境依赖 puppeteer、jest、jest-puppeteer，如果检查到已经安装，则不会进行二次安装
+
+# 判断是否安装了node
+if ! type node >/dev/null 2>&1; then
+    echo -e "\033[31m node 未安装 \033[0m"
+    exit
+fi
+
+# 判断是否安装了npm
+if ! type npm >/dev/null 2>&1; then
+    echo -e "\033[31m npm 未安装 \033[0m"
+    exit
+fi
+
+# MODE be 'js_infer'
+MODE=$1
+# js_infer MODE , load model file and convert model to js_infer
+if [ ${MODE} != "js_infer" ];then
+    echo "Please change mode to 'js_infer'"
+    exit
+fi
+
+
+# saved_model_name
+det_saved_model_name=ch_PP-OCRv2_det_infer
+rec_saved_model_name=ch_PP-OCRv2_rec_infer
+
+# model_path
+model_path=test_tipc/web/models/
+
+rm -rf $model_path
+
+echo ${model_path}${det_saved_model_name}
+echo ${model_path}${rec_saved_model_name}
+
+# download ocr_det inference model
+wget -nc -P $model_path https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_det_infer.tar
+cd $model_path && tar xf ch_PP-OCRv2_det_infer.tar && cd ../../../
+
+# download ocr_rec inference model
+wget -nc -P $model_path https://paddlejs.bj.bcebos.com/models/ch_PP-OCRv2_rec_infer.tar
+cd $model_path && tar xf ch_PP-OCRv2_rec_infer.tar && cd ../../../
+
+MYDIR=`pwd`
+echo $MYDIR
+
+pip3 install paddlejsconverter
+
+# convert inference model to web model: model.json、chunk.dat
+paddlejsconverter \
+   --modelPath=$model_path$det_saved_model_name/inference.pdmodel \
+   --paramPath=$model_path$det_saved_model_name/inference.pdiparams \
+   --outputDir=$model_path$det_saved_model_name/ \
+
+paddlejsconverter \
+   --modelPath=$model_path$rec_saved_model_name/inference.pdmodel \
+   --paramPath=$model_path$rec_saved_model_name/inference.pdiparams \
+   --outputDir=$model_path$rec_saved_model_name/ \
+
+# always install latest ocr sdk
+cd test_tipc/web
+echo -e "\033[33m Installing the latest ocr sdk... \033[0m"
+npm install @paddlejs-models/ocr@latest
+npm info @paddlejs-models/ocr
+echo -e "\033[32m The latest ocr sdk installed completely.!~ \033[0m"
+
+# install dependencies
+if [ `npm list --dept 0 | grep puppeteer | wc -l` -ne 0 ] && [ `npm list --dept 0 | grep jest | wc -l` -ne 0 ];then
+   echo -e "\033[32m Dependencies have installed \033[0m"
+else
+   echo -e "\033[33m Installing dependencies ... \033[0m"
+   npm install jest jest-puppeteer puppeteer
+   echo -e "\033[32m Dependencies installed completely.!~ \033[0m"
+fi
+
+# del package-lock.json
+rm package-lock.json

test_tipc/test_inference_js.sh

+#!/bin/bash
+
+set -o errexit
+set -o nounset
+
+cd test_tipc/web
+# run ocr test in chrome
+./node_modules/.bin/jest --config ./jest.config.js

test_tipc/test_train_inference_python.sh

@@ -183,7 +183,7 @@ function func_inference(){
                     if [[ ${precision} =~ "fp16" || ${precision} =~ "int8" ]] && [ ${use_trt} = "False" ]; then
                         continue
                     fi
-                    if [[ ${use_trt} = "False" || ${precision} =~ "int8" ]] && [ ${_flag_quant} = "True" ]; then
+                    if [[ ${use_trt} = "False" && ${precision} =~ "int8" ]] && [ ${_flag_quant} = "True" ]; then
                         continue
                     fi
                     for batch_size in ${batch_size_list[*]}; do
@@ -227,7 +227,12 @@ if [ ${MODE} = "whole_infer" ] || [ ${MODE} = "klquant_whole_infer" ]; then
     for infer_model in ${infer_model_dir_list[*]}; do
         # run export
         if [ ${infer_run_exports[Count]} != "null" ];then
-            save_infer_dir=$(dirname $infer_model)
+            if [ ${MODE} = "klquant_whole_infer" ]; then
+                save_infer_dir="${infer_model}_klquant"
+            fi
+            if [ ${MODE} = "whole_infer" ]; then
+                save_infer_dir="${infer_model}"
+            fi
             set_export_weight=$(func_set_params "${export_weight}" "${infer_model}")
             set_save_infer_key=$(func_set_params "${save_infer_key}" "${save_infer_dir}")
             export_cmd="${python} ${infer_run_exports[Count]} ${set_export_weight} ${set_save_infer_key}"
@@ -259,7 +264,6 @@ else
             env=""
         elif [ ${#gpu} -le 1 ];then
             env="export CUDA_VISIBLE_DEVICES=${gpu}"
-            eval ${env}
         elif [ ${#gpu} -le 15 ];then
             IFS=","
             array=(${gpu})
@@ -280,6 +284,7 @@ else
                 set_amp_config=" "
             fi          
             for trainer in ${trainer_list[*]}; do 
+                eval ${env}
                 flag_quant=False
                 if [ ${trainer} = ${pact_key} ]; then
                     run_train=${pact_trainer}
@@ -332,7 +337,6 @@ else
                     cmd="${python} -m paddle.distributed.launch --ips=${ips} --gpus=${gpu} ${run_train} ${set_use_gpu} ${set_save_model} ${set_pretrain} ${set_epoch} ${set_autocast} ${set_batchsize} ${set_train_params1} ${set_amp_config}"
                 fi
                 # run train
-                eval "unset CUDA_VISIBLE_DEVICES"
                 eval $cmd
                 status_check $? "${cmd}" "${status_log}"
 

test_tipc/web/expect.json

+{
+    "text":  [
+        "纯臻营养护发素",
+        "产品信息/参数",
+        "（45元/每公斤，100公斤起订）",
+        "每瓶22元，1000瓶起订）",
+        "【品牌】：代加工方式/OEMODM",
+        "【品名】：纯臻营养护发素",
+        "【产品编号】：YM-X-3011",
+        "ODMOEM",
+        "【净含量】：220ml",
+        "【适用人群】：适合所有肤质",
+        "【主要成分】：鲸蜡硬脂醇、燕麦β-葡聚",
+        "糖、椰油酰胺丙基甜菜碱、泛醌",
+        "（成品包材）",
+        "【主要功能】：可紧致头发磷层，从而达到",
+        "即时持久改善头发光泽的效果，给干燥的头",
+        "发足够的滋养"
+    ]
+}

test_tipc/web/index.html

+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <meta http-equiv="X-UA-Compatible" content="ie=edge">
+    <title>ocr test</title>
+</head>
+<body>
+    <img id="ocr" src="./test.jpg" />
+</body>
+<script src="./node_modules/@paddlejs-models/ocr/lib/index.js"></script>
+</html>
\ No newline at end of file

test_tipc/web/index.test.js

+const expectData = require('./expect.json');
+
+describe('e2e test ocr model', () => {
+
+    beforeAll(async () => {
+        await page.goto(PATH);
+    });
+
+    it('ocr infer and diff test', async () => {
+        page.on('console', msg => console.log('PAGE LOG:', msg.text()));
+
+        const text = await page.evaluate(async () => {
+            const $ocr = document.querySelector('#ocr');
+            const ocr = paddlejs['ocr'];
+            await ocr.init('./models/ch_PP-OCRv2_det_infer', './models/ch_PP-OCRv2_rec_infer');
+            const res = await ocr.recognize($ocr);
+            return res.text;
+        });
+        // 模型文字识别结果与预期结果diff的字符数
+        let diffNum = 0;
+        // 文本框字符串相似度
+        let similarity = 0;
+        // 预期字符diff数
+        const expectedDiffNum = 10;
+        // 预期文本框字符串相似度
+        const expecteSimilarity = 0.9;
+        // 预期文本内容
+        const expectResult = expectData.text;
+
+        expectResult && expectResult.forEach((item, index) => {
+            const word = text[index];
+            // 逐字符对比
+            for(let i = 0; i < item.length; i++) {
+                if (item[i] !== word[i]) {
+                    console.log('expect: ', item[i], ' word: ', word[i]);
+                    diffNum++;
+                }
+            }
+            // 文本框字符串相似度对比
+            const s = similar(item, word);
+            similarity += s;
+        });
+
+        similarity = similarity / expectResult.length;
+
+        expect(diffNum).toBeLessThanOrEqual(expectedDiffNum);
+
+        expect(similarity).toBeGreaterThanOrEqual(expecteSimilarity);
+
+        function similar(string, expect) {
+            if (!string || !expect) {
+                return 0;
+            }
+            const length = string.length > expect.length ? string.length : expect.length;
+            const n = string.length;
+            const m = expect.length;
+            let data = [];
+            const min = (a, b, c) => {
+                return a < b ? (a < c ? a : c) : (b < c ? b : c);
+            };
+            let i, j, si, ej, cost;
+            if (n === 0) return m;
+            if (m === 0) return n;
+            for (i = 0; i <= n; i++) {
+                data[i] = [];
+                [i][0] = i
+            }
+            for (j = 0; j <= m; j++) {
+                data[0][j] = j;
+            }
+            for (i = 1; i <= n; i++) {
+                si = string.charAt(i - 1);
+                for (j = 1; j <= m; j++) {
+                    ej = expect.charAt(j - 1);
+                    cost = si === ej ? 0 : 1;
+                    data[i][j] = min(data[i - 1][j] + 1, data[i][j - 1] + 1, data[i - 1][j - 1] + cost);
+                }
+            }
+            return (1 - data[n][m] / length);
+        }
+    });
+});

test_tipc/web/jest-puppeteer.config.js

+// jest-puppeteer.config.js
+module.exports = {
+    launch: {
+        headless: false,
+        product: 'chrome'
+    },
+    browserContext: 'default',
+    server: {
+        command: 'python3 -m http.server 9811',
+        port: 9811,
+        launchTimeout: 10000,
+        debug: true
+    }
+};

test_tipc/web/jest.config.js

+// For a detailed explanation regarding each configuration property and type check, visit:
+// https://jestjs.io/docs/en/configuration.html
+
+module.exports = {
+    preset: 'jest-puppeteer',
+    // All imported modules in your tests should be mocked automatically
+    // automock: false,
+
+    // Automatically clear mock calls and instances between every test
+    clearMocks: true,
+
+    // An object that configures minimum threshold enforcement for coverage results
+    // coverageThreshold: undefined,
+
+    // A set of global variables that need to be available in all test environments
+    globals: {
+        PATH: 'http://localhost:9811'
+    },
+
+    // The maximum amount of workers used to run your tests. Can be specified as % or a number. E.g. maxWorkers: 10% will use 10% of your CPU amount + 1 as the maximum worker number. maxWorkers: 2 will use a maximum of 2 workers.
+    // maxWorkers: "50%",
+
+    // An array of directory names to be searched recursively up from the requiring module's location
+    // moduleDirectories: [
+    //   "node_modules"
+    // ],
+
+    // An array of file extensions your modules use
+    moduleFileExtensions: [
+        'js',
+        'json',
+        'jsx',
+        'ts',
+        'tsx',
+        'node'
+    ],
+
+
+    // The root directory that Jest should scan for tests and modules within
+    // rootDir: undefined,
+
+    // A list of paths to directories that Jest should use to search for files in
+    roots: [
+        '<rootDir>'
+    ],
+
+    // Allows you to use a custom runner instead of Jest's default test runner
+    // runner: "jest-runner",
+
+    // The paths to modules that run some code to configure or set up the testing environment before each test
+    // setupFiles: [],
+
+    // A list of paths to modules that run some code to configure or set up the testing framework before each test
+    // setupFilesAfterEnv: [],
+
+    // The number of seconds after which a test is considered as slow and reported as such in the results.
+    // slowTestThreshold: 5,
+
+    // A list of paths to snapshot serializer modules Jest should use for snapshot testing
+    // snapshotSerializers: [],
+
+    // The test environment that will be used for testing
+    // testEnvironment: 'jsdom',
+
+    // Options that will be passed to the testEnvironment
+    // testEnvironmentOptions: {},
+
+    // An array of regexp pattern strings that are matched against all test paths, matched tests are skipped
+    testPathIgnorePatterns: [
+        '/node_modules/'
+    ],
+
+    // The regexp pattern or array of patterns that Jest uses to detect test files
+    testRegex: '.(.+)\\.test\\.(js|ts)$',
+
+    // This option allows the use of a custom results processor
+    // testResultsProcessor: undefined,
+
+    // This option allows use of a custom test runner
+    // testRunner: "jest-circus/runner",
+
+    // This option sets the URL for the jsdom environment. It is reflected in properties such as location.href
+    testURL: 'http://localhost:9898/',
+
+    // Setting this value to "fake" allows the use of fake timers for functions such as "setTimeout"
+    // timers: "real",
+
+    // A map from regular expressions to paths to transformers
+    transform: {
+        '^.+\\.js$': 'babel-jest'
+    },
+
+    // An array of regexp pattern strings that are matched against all source file paths, matched files will skip transformation
+    transformIgnorePatterns: [
+        '/node_modules/',
+        '\\.pnp\\.[^\\/]+$'
+    ],
+
+    // An array of regexp pattern strings that are matched against all modules before the module loader will automatically return a mock for them
+    // unmockedModulePathPatterns: undefined,
+
+    // Indicates whether each individual test should be reported during the run
+    verbose: true,
+
+    // An array of regexp patterns that are matched against all source file paths before re-running tests in watch mode
+    // watchPathIgnorePatterns: [],
+
+    // Whether to use watchman for file crawling
+    // watchman: true,
+    testTimeout: 50000
+};

tools/infer/predict_det.py

@@ -101,16 +101,21 @@ class TextDetector(object):
         else:
             logger.info("unknown det_algorithm:{}".format(self.det_algorithm))
             sys.exit(0)
+
+        self.preprocess_op = create_operators(pre_process_list)
+        self.postprocess_op = build_post_process(postprocess_params)
+        self.predictor, self.input_tensor, self.output_tensors, self.config = utility.create_predictor(
+            args, 'det', logger)
+
         if self.use_onnx:
+            img_h, img_w = self.input_tensor.shape[2:]
+            if img_h is not None and img_w is not None and img_h > 0 and img_w > 0:
                 pre_process_list[0] = {
                     'DetResizeForTest': {
-                    'image_shape': [640, 640]
+                        'image_shape': [img_h, img_w]
                     }
                 }
         self.preprocess_op = create_operators(pre_process_list)
-        self.postprocess_op = build_post_process(postprocess_params)
-        self.predictor, self.input_tensor, self.output_tensors, self.config = utility.create_predictor(
-            args, 'det', logger)
 
         if args.benchmark:
             import auto_log

tools/infer/predict_rec.py

@@ -109,7 +109,10 @@ class TextRecognizer(object):
         assert imgC == img.shape[2]
         imgW = int((32 * max_wh_ratio))
         if self.use_onnx:
-            imgW = 100
+            w = self.input_tensor.shape[3:][0]
+            if w is not None and w > 0:
+                imgW = w
+
         h, w = img.shape[:2]
         ratio = w / float(h)
         if math.ceil(imgH * ratio) > imgW:

tools/infer/utility.py

@@ -15,6 +15,7 @@
 import argparse
 import os
 import sys
+import platform
 import cv2
 import numpy as np
 import paddle
@@ -313,6 +314,10 @@ def create_predictor(args, mode, logger):
 
 
 def get_infer_gpuid():
+    sysstr = platform.system()
+    if sysstr == "Windows":
+        return 0
+
     if not paddle.fluid.core.is_compiled_with_rocm():
         cmd = "env | grep CUDA_VISIBLE_DEVICES"
     else: