prepare.sh 11.8 KB
Newer Older
LDOUBLEV's avatar
LDOUBLEV committed
1
2
#!/bin/bash
FILENAME=$1
3

LDOUBLEV's avatar
rename  
LDOUBLEV committed
4
5
6
# MODE be one of ['lite_train_lite_infer' 'lite_train_whole_infer' 'whole_train_whole_infer',  
#                 'whole_infer', 'klquant_whole_infer',
#                 'cpp_infer', 'serving_infer',  'lite_infer']
LDOUBLEV's avatar
LDOUBLEV committed
7

LDOUBLEV's avatar
LDOUBLEV committed
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
MODE=$2

dataline=$(cat ${FILENAME})

# parser params
IFS=$'\n'
lines=(${dataline})
function func_parser_key(){
    strs=$1
    IFS=":"
    array=(${strs})
    tmp=${array[0]}
    echo ${tmp}
}
function func_parser_value(){
    strs=$1
    IFS=":"
    array=(${strs})
    tmp=${array[1]}
    echo ${tmp}
}
IFS=$'\n'
# The training params
LDOUBLEV's avatar
LDOUBLEV committed
31
model_name=$(func_parser_value "${lines[1]}")
LDOUBLEV's avatar
LDOUBLEV committed
32

LDOUBLEV's avatar
LDOUBLEV committed
33
trainer_list=$(func_parser_value "${lines[14]}")
Double_V's avatar
Double_V committed
34

Double_V's avatar
Double_V committed
35
36
# MODE be one of ['lite_train_infer' 'whole_infer' 'whole_train_infer']
MODE=$2
LDOUBLEV's avatar
LDOUBLEV committed
37

LDOUBLEV's avatar
rename  
LDOUBLEV committed
38
if [ ${MODE} = "lite_train_lite_infer" ];then
Double_V's avatar
Double_V committed
39
40
    # pretrain lite train data
    wget -nc -P  ./pretrain_models/ https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV3_large_x0_5_pretrained.pdparams
LDOUBLEV's avatar
LDOUBLEV committed
41
    wget -nc -P ./pretrain_models/  https://paddleocr.bj.bcebos.com/dygraph_v2.0/en/det_mv3_db_v2.0_train.tar
LDOUBLEV's avatar
LDOUBLEV committed
42
43
44
45
    if [ ${model_name} == "PPOCRv2_ocr_det" ]; then
        wget -nc -P ./pretrain_models/ https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_det_distill_train.tar
        cd ./pretrain_models/ && tar xf ch_PP-OCRv2_det_distill_train.tar && cd ../
    fi
LDOUBLEV's avatar
LDOUBLEV committed
46
    cd ./pretrain_models/ && tar xf det_mv3_db_v2.0_train.tar && cd ../
Double_V's avatar
Double_V committed
47
    rm -rf ./train_data/icdar2015
tink2123's avatar
tink2123 committed
48
    rm -rf ./train_data/ic15_data
Double_V's avatar
Double_V committed
49
    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/icdar2015_lite.tar
tink2123's avatar
tink2123 committed
50
    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ic15_data.tar # todo change to bcebos
tink2123's avatar
tink2123 committed
51
    wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/rec_inference.tar
LDOUBLEV's avatar
LDOUBLEV committed
52
53
    wget -nc -P ./deploy/slim/prune https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/sen.pickle
    
tink2123's avatar
tink2123 committed
54
    cd ./train_data/ && tar xf icdar2015_lite.tar && tar xf ic15_data.tar
Double_V's avatar
Double_V committed
55
56
    ln -s ./icdar2015_lite ./icdar2015
    cd ../
tink2123's avatar
tink2123 committed
57
    cd ./inference && tar xf rec_inference.tar && cd ../
LDOUBLEV's avatar
rename  
LDOUBLEV committed
58
elif [ ${MODE} = "whole_train_whole_infer" ];then
Double_V's avatar
Double_V committed
59
60
    wget -nc -P  ./pretrain_models/ https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV3_large_x0_5_pretrained.pdparams
    rm -rf ./train_data/icdar2015
tink2123's avatar
tink2123 committed
61
    rm -rf ./train_data/ic15_data
Double_V's avatar
Double_V committed
62
    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/icdar2015.tar
tink2123's avatar
tink2123 committed
63
64
    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ic15_data.tar
    cd ./train_data/ && tar xf icdar2015.tar && tar xf ic15_data.tar && cd ../
LDOUBLEV's avatar
LDOUBLEV committed
65
66
67
68
    if [ ${model_name} == "PPOCRv2_ocr_det" ]; then
        wget -nc -P ./pretrain_models/ https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_det_distill_train.tar
        cd ./pretrain_models/ && tar xf ch_PP-OCRv2_det_distill_train.tar && cd ../
    fi
LDOUBLEV's avatar
rename  
LDOUBLEV committed
69
elif [ ${MODE} = "lite_train_whole_infer" ];then
Double_V's avatar
Double_V committed
70
71
    wget -nc -P  ./pretrain_models/ https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV3_large_x0_5_pretrained.pdparams
    rm -rf ./train_data/icdar2015
tink2123's avatar
tink2123 committed
72
    rm -rf ./train_data/ic15_data
Double_V's avatar
Double_V committed
73
    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/icdar2015_infer.tar
tink2123's avatar
tink2123 committed
74
75
    wget -nc -P ./train_data/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ic15_data.tar
    cd ./train_data/ && tar xf icdar2015_infer.tar && tar xf ic15_data.tar
Double_V's avatar
Double_V committed
76
77
    ln -s ./icdar2015_infer ./icdar2015
    cd ../
LDOUBLEV's avatar
LDOUBLEV committed
78
79
80
81
    if [ ${model_name} == "PPOCRv2_ocr_det" ]; then
        wget -nc -P ./pretrain_models/ https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_det_distill_train.tar
        cd ./pretrain_models/ && tar xf ch_PP-OCRv2_det_distill_train.tar && cd ../
    fi
LDOUBLEV's avatar
rename  
LDOUBLEV committed
82
elif [ ${MODE} = "whole_infer" ];then
tink2123's avatar
tink2123 committed
83
    if [ ${model_name} = "ocr_det" ]; then
LDOUBLEV's avatar
LDOUBLEV committed
84
        eval_model_name="ch_ppocr_mobile_v2.0_det_train"
tink2123's avatar
tink2123 committed
85
        rm -rf ./train_data/icdar2015
LDOUBLEV's avatar
LDOUBLEV committed
86
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar
LDOUBLEV's avatar
LDOUBLEV committed
87
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_det_train.tar
Double_V's avatar
Double_V committed
88
        cd ./inference && tar xf ${eval_model_name}.tar && tar xf ch_det_data_50.tar && cd ../
LDOUBLEV's avatar
LDOUBLEV committed
89
    elif [ ${model_name} = "ocr_server_det" ]; then
LDOUBLEV's avatar
LDOUBLEV committed
90
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_server_v2.0_det_train.tar
LDOUBLEV's avatar
LDOUBLEV committed
91
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar
LDOUBLEV's avatar
LDOUBLEV committed
92
        cd ./inference && tar xf ch_ppocr_server_v2.0_det_train.tar && tar xf ch_det_data_50.tar && cd ../
tink2123's avatar
tink2123 committed
93
    elif  [ ${model_name} = "ocr_system_mobile" ]; then
LDOUBLEV's avatar
LDOUBLEV committed
94
95
96
97
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_det_infer.tar
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_rec_infer.tar
        cd ./inference && tar xf ch_ppocr_mobile_v2.0_det_infer.tar && tar xf ch_ppocr_mobile_v2.0_rec_infer.tar && tar xf ch_det_data_50.tar && cd ../
tink2123's avatar
tink2123 committed
98
99
100
101
102
    elif  [ ${model_name} = "ocr_system_server" ]; then
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_server_v2.0_det_infer.tar
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_server_v2.0_rec_infer.tar
        cd ./inference && tar xf ch_ppocr_server_v2.0_det_infer.tar && tar xf ch_ppocr_server_v2.0_rec_infer.tar && tar xf ch_det_data_50.tar && cd ../
103
    elif [ ${model_name} = "ocr_rec" ]; then
tink2123's avatar
tink2123 committed
104
        rm -rf ./train_data/ic15_data
tink2123's avatar
tink2123 committed
105
        eval_model_name="ch_ppocr_mobile_v2.0_rec_infer"
tink2123's avatar
tink2123 committed
106
        wget -nc -P ./inference/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/rec_inference.tar
tink2123's avatar
tink2123 committed
107
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_rec_infer.tar
tink2123's avatar
tink2123 committed
108
        cd ./inference && tar xf ${eval_model_name}.tar && tar xf rec_inference.tar && cd ../
109
110
111
112
113
114
    elif [ ${model_name} = "ocr_server_rec" ]; then
        rm -rf ./train_data/ic15_data
        eval_model_name="ch_ppocr_server_v2.0_rec_infer"
        wget -nc -P ./inference/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/rec_inference.tar
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_server_v2.0_rec_infer.tar
        cd ./inference && tar xf ${eval_model_name}.tar && tar xf rec_inference.tar && cd ../
Double_V's avatar
Double_V committed
115
    fi 
116

LDOUBLEV's avatar
LDOUBLEV committed
117
118
119
120
121
    elif [ ${model_name} = "PPOCRv2_ocr_det" ]; then
        eval_model_name="ch_PP-OCRv2_det_infer"
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar
        wget -nc -P ./inference/ https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_det_infer.tar
        cd ./inference && tar xf ${eval_model_name}.tar && tar xf ch_det_data_50.tar && cd ../
LDOUBLEV's avatar
fix bug  
LDOUBLEV committed
122
    fi
123

LDOUBLEV's avatar
fix bug  
LDOUBLEV committed
124
if [ ${MODE} = "klquant_whole_infer" ]; then
LDOUBLEV's avatar
LDOUBLEV committed
125
126
127
128
129
    if [ ${model_name} = "ocr_det" ]; then
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_det_infer.tar
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar
        cd ./inference && tar xf ch_ppocr_mobile_v2.0_det_infer.tar && tar xf ch_det_data_50.tar && cd ../
    fi
LDOUBLEV's avatar
LDOUBLEV committed
130
131
132
133
134
135
    if [ ${model_name} = "PPOCRv2_ocr_det" ]; then
        eval_model_name="ch_PP-OCRv2_det_infer"
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar
        wget -nc -P ./inference/ https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_det_infer.tar
        cd ./inference && tar xf ${eval_model_name}.tar && tar xf ch_det_data_50.tar && cd ../
    fi 
LDOUBLEV's avatar
fix bug  
LDOUBLEV committed
136
137
138
fi

if [ ${MODE} = "cpp_infer" ];then
MissPenguin's avatar
MissPenguin committed
139
140
141
142
143
144
145
146
147
148
149
150
151
152
    if [ ${model_name} = "ocr_det" ]; then
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_det_infer.tar
        cd ./inference && tar xf ch_ppocr_mobile_v2.0_det_infer.tar && tar xf ch_det_data_50.tar && cd ../
    elif [ ${model_name} = "ocr_rec" ]; then
        wget -nc -P ./inference/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/rec_inference.tar
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_rec_infer.tar
        cd ./inference && tar xf ch_ppocr_mobile_v2.0_rec_infer.tar && tar xf rec_inference.tar && cd ../
    elif  [ ${model_name} = "ocr_system" ]; then
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_det_infer.tar
        wget -nc -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/ch_det_data_50.tar
        wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_rec_infer.tar
        cd ./inference && tar xf ch_ppocr_mobile_v2.0_det_infer.tar && tar xf ch_ppocr_mobile_v2.0_rec_infer.tar && tar xf ch_det_data_50.tar && cd ../
    fi 
Double_V's avatar
Double_V committed
153
fi
tink2123's avatar
tink2123 committed
154

tink2123's avatar
tink2123 committed
155
156
157
if [ ${MODE} = "serving_infer" ];then
    # prepare serving env
    python_name=$(func_parser_value "${lines[2]}")
158
159
    wget https://paddle-serving.bj.bcebos.com/chain/paddle_serving_server_gpu-0.0.0.post101-py3-none-any.whl
    ${python_name} -m pip install install paddle_serving_server_gpu-0.0.0.post101-py3-none-any.whl
tink2123's avatar
tink2123 committed
160
    ${python_name} -m pip install paddle_serving_client==0.6.1
161
    ${python_name} -m pip install paddle-serving-app==0.6.3
tink2123's avatar
tink2123 committed
162
163
    wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_det_infer.tar
    wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_rec_infer.tar
164
165
    wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_server_v2.0_det_infer.tar
    wget -nc  -P ./inference https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_server_v2.0_rec_infer.tar
tink2123's avatar
tink2123 committed
166
    cd ./inference && tar xf ch_ppocr_mobile_v2.0_det_infer.tar && tar xf ch_ppocr_mobile_v2.0_rec_infer.tar && tar xf ch_ppocr_server_v2.0_rec_infer.tar && tar xf ch_ppocr_server_v2.0_det_infer.tar && cd ../
tink2123's avatar
tink2123 committed
167
fi
cuicheng01's avatar
cuicheng01 committed
168
169
170
171
172
173
174
175
176
177
178
179
180
181


if [ ${MODE} = "lite_infer" ];then    
    # prepare lite nb model and test data
    current_dir=${PWD}
    wget -nc  -P ./models https://paddleocr.bj.bcebos.com/dygraph_v2.0/lite/ch_ppocr_mobile_v2.0_det_opt.nb
    wget -nc  -P ./models https://paddleocr.bj.bcebos.com/dygraph_v2.0/lite/ch_ppocr_mobile_v2.0_det_slim_opt.nb
    wget -nc  -P ./test_data https://paddleocr.bj.bcebos.com/dygraph_v2.0/test/icdar2015_lite.tar
    cd ./test_data && tar -xf icdar2015_lite.tar && rm icdar2015_lite.tar && cd ../
    # prepare lite env
    export http_proxy=http://172.19.57.45:3128
    export https_proxy=http://172.19.57.45:3128
    paddlelite_url=https://github.com/PaddlePaddle/Paddle-Lite/releases/download/v2.9/inference_lite_lib.android.armv8.gcc.c++_shared.with_extra.with_cv.tar.gz
    paddlelite_zipfile=$(echo $paddlelite_url | awk -F "/" '{print $NF}')
cuicheng01's avatar
cuicheng01 committed
182
    paddlelite_file=${paddlelite_zipfile:0:66}
cuicheng01's avatar
cuicheng01 committed
183
184
185
186
187
188
189
    wget ${paddlelite_url}
    tar -xf ${paddlelite_zipfile}
    mkdir -p  ${paddlelite_file}/demo/cxx/ocr/test_lite
    mv models test_data ${paddlelite_file}/demo/cxx/ocr/test_lite
    cp ppocr/utils/ppocr_keys_v1.txt deploy/lite/config.txt ${paddlelite_file}/demo/cxx/ocr/test_lite
    cp ./deploy/lite/* ${paddlelite_file}/demo/cxx/ocr/
    cp ${paddlelite_file}/cxx/lib/libpaddle_light_api_shared.so ${paddlelite_file}/demo/cxx/ocr/test_lite
cuicheng01's avatar
cuicheng01 committed
190
    cp test_tipc/configs/ppocr_det_mobile_params.txt test_tipc/test_lite.sh test_tipc/common_func.sh ${paddlelite_file}/demo/cxx/ocr/test_lite
cuicheng01's avatar
cuicheng01 committed
191
192
193
194
195
196
197
198
199
200
201
    cd ${paddlelite_file}/demo/cxx/ocr/
    git clone https://github.com/LDOUBLEV/AutoLog.git
    unset http_proxy
    unset https_proxy
    make -j
    sleep 1
    make -j
    cp ocr_db_crnn test_lite && cp test_lite/libpaddle_light_api_shared.so test_lite/libc++_shared.so
    tar -cf test_lite.tar ./test_lite && cp test_lite.tar ${current_dir} && cd ${current_dir}
fi