prepare.sh 12.8 KB
Newer Older
LDOUBLEV's avatar
LDOUBLEV committed
1
2
#!/bin/bash
FILENAME=$1
3

LDOUBLEV's avatar
rename  
LDOUBLEV committed
4
5
6
# MODE be one of ['lite_train_lite_infer' 'lite_train_whole_infer' 'whole_train_whole_infer',  
#                 'whole_infer', 'klquant_whole_infer',
#                 'cpp_infer', 'serving_infer',  'lite_infer']
LDOUBLEV's avatar
LDOUBLEV committed
7

LDOUBLEV's avatar
LDOUBLEV committed
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
MODE=$2

dataline=$(cat ${FILENAME})

# parser params
IFS=$'\n'
lines=(${dataline})
function func_parser_key(){
    strs=$1
    IFS=":"
    array=(${strs})
    tmp=${array[0]}
    echo ${tmp}
}
function func_parser_value(){
    strs=$1
    IFS=":"
    array=(${strs})
    tmp=${array[1]}
    echo ${tmp}
}
IFS=$'\n'
# The training params
LDOUBLEV's avatar
LDOUBLEV committed
31
model_name=$(func_parser_value "${lines[1]}")
LDOUBLEV's avatar
LDOUBLEV committed
32

LDOUBLEV's avatar
LDOUBLEV committed
33
trainer_list=$(func_parser_value "${lines[14]}")
Double_V's avatar
Double_V committed
34

Double_V's avatar
Double_V committed
35
36
# MODE be one of ['lite_train_infer' 'whole_infer' 'whole_train_infer']
MODE=$2
LDOUBLEV's avatar
LDOUBLEV committed
37

LDOUBLEV's avatar
rename  
LDOUBLEV committed
38
if [ ${MODE} = "lite_train_lite_infer" ];then
Double_V's avatar
Double_V committed
39
    # pretrain lite train data
LDOUBLEV's avatar
LDOUBLEV committed
40
41
    wget -nc -P  ./pretrain_models/ https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV3_large_x0_5_pretrained.pdparams  --no-check-certificate
    wget -nc -P ./pretrain_models/  https://paddleocr.bj.bcebos.com/dygraph_v2.0/en/det_mv3_db_v2.0_train.tar  --no-check-certificate
LDOUBLEV's avatar
LDOUBLEV committed
42
    if [ ${model_name} == "PPOCRv2_ocr_det" ]; then
LDOUBLEV's avatar
LDOUBLEV committed
43
        wget -nc -P ./pretrain_models/ https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_det_distill_train.tar --no-check-certificate
LDOUBLEV's avatar
LDOUBLEV committed
44
45
        cd ./pretrain_models/ && tar xf ch_PP-OCRv2_det_distill_train.tar && cd ../
    fi
LDOUBLEV's avatar
LDOUBLEV committed
46
    cd ./pretrain_models/ && tar xf det_mv3_db_v2.0_train.tar && cd ../
Double_V's avatar
Double_V committed
47
    rm -rf ./train_data/icdar2015
tink2123's avatar
tink2123 committed
48
    rm -rf ./train_data/ic15_data
LDOUBLEV's avatar
LDOUBLEV committed
49
50
51
52
    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/icdar2015_lite.tar --no-check-certificate
    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ic15_data.tar --no-check-certificate
    wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/rec_inference.tar --no-check-certificate
    wget -nc -P ./deploy/slim/prune https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/sen.pickle --no-check-certificate
LDOUBLEV's avatar
LDOUBLEV committed
53
    
tink2123's avatar
tink2123 committed
54
    cd ./train_data/ && tar xf icdar2015_lite.tar && tar xf ic15_data.tar
Double_V's avatar
Double_V committed
55
56
    ln -s ./icdar2015_lite ./icdar2015
    cd ../
tink2123's avatar
tink2123 committed
57
    cd ./inference && tar xf rec_inference.tar && cd ../
LDOUBLEV's avatar
rename  
LDOUBLEV committed
58
elif [ ${MODE} = "whole_train_whole_infer" ];then
LDOUBLEV's avatar
LDOUBLEV committed
59
    wget -nc -P  ./pretrain_models/ https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV3_large_x0_5_pretrained.pdparams --no-check-certificate
Double_V's avatar
Double_V committed
60
    rm -rf ./train_data/icdar2015
tink2123's avatar
tink2123 committed
61
    rm -rf ./train_data/ic15_data
LDOUBLEV's avatar
LDOUBLEV committed
62
63
    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/icdar2015.tar --no-check-certificate
    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ic15_data.tar --no-check-certificate
tink2123's avatar
tink2123 committed
64
    cd ./train_data/ && tar xf icdar2015.tar && tar xf ic15_data.tar && cd ../
LDOUBLEV's avatar
LDOUBLEV committed
65
    if [ ${model_name} == "PPOCRv2_ocr_det" ]; then
LDOUBLEV's avatar
LDOUBLEV committed
66
        wget -nc -P ./pretrain_models/ https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_det_distill_train.tar --no-check-certificate
LDOUBLEV's avatar
LDOUBLEV committed
67
68
        cd ./pretrain_models/ && tar xf ch_PP-OCRv2_det_distill_train.tar && cd ../
    fi
LDOUBLEV's avatar
rename  
LDOUBLEV committed
69
elif [ ${MODE} = "lite_train_whole_infer" ];then
LDOUBLEV's avatar
LDOUBLEV committed
70
    wget -nc -P  ./pretrain_models/ https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV3_large_x0_5_pretrained.pdparams --no-check-certificate
Double_V's avatar
Double_V committed
71
    rm -rf ./train_data/icdar2015
tink2123's avatar
tink2123 committed
72
    rm -rf ./train_data/ic15_data
LDOUBLEV's avatar
LDOUBLEV committed
73
74
    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/icdar2015_infer.tar --no-check-certificate
    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ic15_data.tar --no-check-certificate
tink2123's avatar
tink2123 committed
75
    cd ./train_data/ && tar xf icdar2015_infer.tar && tar xf ic15_data.tar
Double_V's avatar
Double_V committed
76
77
    ln -s ./icdar2015_infer ./icdar2015
    cd ../
LDOUBLEV's avatar
LDOUBLEV committed
78
    if [ ${model_name} == "PPOCRv2_ocr_det" ]; then
LDOUBLEV's avatar
LDOUBLEV committed
79
        wget -nc -P ./pretrain_models/ https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_det_distill_train.tar --no-check-certificate
LDOUBLEV's avatar
LDOUBLEV committed
80
81
        cd ./pretrain_models/ && tar xf ch_PP-OCRv2_det_distill_train.tar && cd ../
    fi
LDOUBLEV's avatar
rename  
LDOUBLEV committed
82
elif [ ${MODE} = "whole_infer" ];then
tink2123's avatar
tink2123 committed
83
    if [ ${model_name} = "ocr_det" ]; then
LDOUBLEV's avatar
LDOUBLEV committed
84
        eval_model_name="ch_ppocr_mobile_v2.0_det_train"
tink2123's avatar
tink2123 committed
85
        rm -rf ./train_data/icdar2015
LDOUBLEV's avatar
LDOUBLEV committed
86
87
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar --no-check-certificate
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_det_train.tar --no-check-certificate
Double_V's avatar
Double_V committed
88
        cd ./inference && tar xf ${eval_model_name}.tar && tar xf ch_det_data_50.tar && cd ../
LDOUBLEV's avatar
LDOUBLEV committed
89
    elif [ ${model_name} = "ocr_server_det" ]; then
LDOUBLEV's avatar
LDOUBLEV committed
90
91
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_server_v2.0_det_train.tar --no-check-certificate
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar  --no-check-certificate
LDOUBLEV's avatar
LDOUBLEV committed
92
        cd ./inference && tar xf ch_ppocr_server_v2.0_det_train.tar && tar xf ch_det_data_50.tar && cd ../
tink2123's avatar
tink2123 committed
93
    elif  [ ${model_name} = "ocr_system_mobile" ]; then
LDOUBLEV's avatar
LDOUBLEV committed
94
95
96
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_det_infer.tar --no-check-certificate
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar --no-check-certificate
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_rec_infer.tar --no-check-certificate
LDOUBLEV's avatar
LDOUBLEV committed
97
        cd ./inference && tar xf ch_ppocr_mobile_v2.0_det_infer.tar && tar xf ch_ppocr_mobile_v2.0_rec_infer.tar && tar xf ch_det_data_50.tar && cd ../
tink2123's avatar
tink2123 committed
98
    elif  [ ${model_name} = "ocr_system_server" ]; then
LDOUBLEV's avatar
LDOUBLEV committed
99
100
101
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_server_v2.0_det_infer.tar --no-check-certificate
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar --no-check-certificate
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_server_v2.0_rec_infer.tar --no-check-certificate
tink2123's avatar
tink2123 committed
102
        cd ./inference && tar xf ch_ppocr_server_v2.0_det_infer.tar && tar xf ch_ppocr_server_v2.0_rec_infer.tar && tar xf ch_det_data_50.tar && cd ../
103
    elif [ ${model_name} = "ocr_rec" ]; then
tink2123's avatar
tink2123 committed
104
        rm -rf ./train_data/ic15_data
tink2123's avatar
tink2123 committed
105
        eval_model_name="ch_ppocr_mobile_v2.0_rec_infer"
LDOUBLEV's avatar
LDOUBLEV committed
106
107
        wget -nc -P ./inference/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/rec_inference.tar --no-check-certificate
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_rec_infer.tar --no-check-certificate
tink2123's avatar
tink2123 committed
108
        cd ./inference && tar xf ${eval_model_name}.tar && tar xf rec_inference.tar && cd ../
109
110
111
    elif [ ${model_name} = "ocr_server_rec" ]; then
        rm -rf ./train_data/ic15_data
        eval_model_name="ch_ppocr_server_v2.0_rec_infer"
LDOUBLEV's avatar
LDOUBLEV committed
112
113
        wget -nc -P ./inference/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/rec_inference.tar --no-check-certificate
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_server_v2.0_rec_infer.tar --no-check-certificate
114
        cd ./inference && tar xf ${eval_model_name}.tar && tar xf rec_inference.tar && cd ../
Double_V's avatar
Double_V committed
115
    fi 
116

LDOUBLEV's avatar
LDOUBLEV committed
117
118
    elif [ ${model_name} = "PPOCRv2_ocr_det" ]; then
        eval_model_name="ch_PP-OCRv2_det_infer"
LDOUBLEV's avatar
LDOUBLEV committed
119
120
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar --no-check-certificate
        wget -nc -P ./inference/ https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_det_infer.tar --no-check-certificate
LDOUBLEV's avatar
LDOUBLEV committed
121
        cd ./inference && tar xf ${eval_model_name}.tar && tar xf ch_det_data_50.tar && cd ../
LDOUBLEV's avatar
fix bug  
LDOUBLEV committed
122
    fi
123

LDOUBLEV's avatar
fix bug  
LDOUBLEV committed
124
if [ ${MODE} = "klquant_whole_infer" ]; then
LDOUBLEV's avatar
LDOUBLEV committed
125
    if [ ${model_name} = "ocr_det" ]; then
LDOUBLEV's avatar
LDOUBLEV committed
126
127
128
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_det_infer.tar --no-check-certificate
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar  --no-check-certificate
        cd ./inference && tar xf ch_ppocr_mobile_v2.0_det_infer.tar && tar xf ch_det_data_50.tar && cd ../ 
LDOUBLEV's avatar
LDOUBLEV committed
129
    fi
LDOUBLEV's avatar
LDOUBLEV committed
130
131
    if [ ${model_name} = "PPOCRv2_ocr_det" ]; then
        eval_model_name="ch_PP-OCRv2_det_infer"
LDOUBLEV's avatar
LDOUBLEV committed
132
133
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar  --no-check-certificate
        wget -nc -P ./inference/ https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_det_infer.tar  --no-check-certificate
LDOUBLEV's avatar
LDOUBLEV committed
134
135
        cd ./inference && tar xf ${eval_model_name}.tar && tar xf ch_det_data_50.tar && cd ../
    fi 
LDOUBLEV's avatar
fix bug  
LDOUBLEV committed
136
137
138
fi

if [ ${MODE} = "cpp_infer" ];then
MissPenguin's avatar
MissPenguin committed
139
    if [ ${model_name} = "ocr_det" ]; then
LDOUBLEV's avatar
LDOUBLEV committed
140
141
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar  --no-check-certificate
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_det_infer.tar  --no-check-certificate
MissPenguin's avatar
MissPenguin committed
142
143
        cd ./inference && tar xf ch_ppocr_mobile_v2.0_det_infer.tar && tar xf ch_det_data_50.tar && cd ../
    elif [ ${model_name} = "ocr_rec" ]; then
LDOUBLEV's avatar
LDOUBLEV committed
144
145
        wget -nc -P ./inference/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/rec_inference.tar  --no-check-certificate
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_rec_infer.tar  --no-check-certificate
MissPenguin's avatar
MissPenguin committed
146
147
        cd ./inference && tar xf ch_ppocr_mobile_v2.0_rec_infer.tar && tar xf rec_inference.tar && cd ../
    elif  [ ${model_name} = "ocr_system" ]; then
LDOUBLEV's avatar
LDOUBLEV committed
148
149
150
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_det_infer.tar  --no-check-certificate
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar  --no-check-certificate
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_rec_infer.tar  --no-check-certificate
MissPenguin's avatar
MissPenguin committed
151
152
        cd ./inference && tar xf ch_ppocr_mobile_v2.0_det_infer.tar && tar xf ch_ppocr_mobile_v2.0_rec_infer.tar && tar xf ch_det_data_50.tar && cd ../
    fi 
Double_V's avatar
Double_V committed
153
fi
tink2123's avatar
tink2123 committed
154

tink2123's avatar
tink2123 committed
155
156
157
if [ ${MODE} = "serving_infer" ];then
    # prepare serving env
    python_name=$(func_parser_value "${lines[2]}")
158
159
    wget https://paddle-serving.bj.bcebos.com/chain/paddle_serving_server_gpu-0.0.0.post101-py3-none-any.whl
    ${python_name} -m pip install install paddle_serving_server_gpu-0.0.0.post101-py3-none-any.whl
tink2123's avatar
tink2123 committed
160
    ${python_name} -m pip install paddle_serving_client==0.6.1
161
    ${python_name} -m pip install paddle-serving-app==0.6.3
tink2123's avatar
tink2123 committed
162
163
    wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_det_infer.tar
    wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_rec_infer.tar
164
165
    wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_server_v2.0_det_infer.tar
    wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_server_v2.0_rec_infer.tar
tink2123's avatar
tink2123 committed
166
    cd ./inference && tar xf ch_ppocr_mobile_v2.0_det_infer.tar && tar xf ch_ppocr_mobile_v2.0_rec_infer.tar && tar xf ch_ppocr_server_v2.0_rec_infer.tar && tar xf ch_ppocr_server_v2.0_det_infer.tar && cd ../
tink2123's avatar
tink2123 committed
167
fi
cuicheng01's avatar
cuicheng01 committed
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201


if [ ${MODE} = "lite_infer" ];then    
    # prepare lite nb model and test data
    current_dir=${PWD}
    wget -nc  -P ./models https://paddleocr.bj.bcebos.com/dygraph_v2.0/lite/ch_ppocr_mobile_v2.0_det_opt.nb
    wget -nc  -P ./models https://paddleocr.bj.bcebos.com/dygraph_v2.0/lite/ch_ppocr_mobile_v2.0_det_slim_opt.nb
    wget -nc  -P ./test_data https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/icdar2015_lite.tar
    cd ./test_data && tar -xf icdar2015_lite.tar && rm icdar2015_lite.tar && cd ../
    # prepare lite env
    export http_proxy=http://172.19.57.45:3128
    export https_proxy=http://172.19.57.45:3128
    paddlelite_url=https://github.com/PaddlePaddle/Paddle-Lite/releases/download/v2.9/inference_lite_lib.android.armv8.gcc.c++_shared.with_extra.with_cv.tar.gz
    paddlelite_zipfile=$(echo $paddlelite_url | awk -F "/" '{print $NF}')
    paddlelite_file=inference_lite_lib.android.armv8.gcc.c++_shared.with_extra.with_cv
    wget ${paddlelite_url}
    tar -xf ${paddlelite_zipfile}
    mkdir -p  ${paddlelite_file}/demo/cxx/ocr/test_lite
    mv models test_data ${paddlelite_file}/demo/cxx/ocr/test_lite
    cp ppocr/utils/ppocr_keys_v1.txt deploy/lite/config.txt ${paddlelite_file}/demo/cxx/ocr/test_lite
    cp ./deploy/lite/* ${paddlelite_file}/demo/cxx/ocr/
    cp ${paddlelite_file}/cxx/lib/libpaddle_light_api_shared.so ${paddlelite_file}/demo/cxx/ocr/test_lite
    cp PTDN/configs/ppocr_det_mobile_params.txt PTDN/test_lite.sh PTDN/common_func.sh ${paddlelite_file}/demo/cxx/ocr/test_lite
    cd ${paddlelite_file}/demo/cxx/ocr/
    git clone https://github.com/LDOUBLEV/AutoLog.git
    unset http_proxy
    unset https_proxy
    make -j
    sleep 1
    make -j
    cp ocr_db_crnn test_lite && cp test_lite/libpaddle_light_api_shared.so test_lite/libc++_shared.so
    tar -cf test_lite.tar ./test_lite && cp test_lite.tar ${current_dir} && cd ${current_dir}
fi