Skip to content

GitLab

Unverified Commit cf613b13 authored by Bin Lu's avatar Bin Lu Committed by GitHub
Browse files

Merge branch 'PaddlePaddle:dygraph' into dygraph

parents 8fe6209d 732fa778
Hide whitespace changes
Inline Side-by-side
Showing with 64 additions and 39 deletions
test_tipc/test_train_inference_python.sh
View file @ cf613b13
......@@ -284,7 +284,6 @@ else
set_amp_config=" "
fi
for trainer in ${trainer_list[*]}; do
eval ${env}
flag_quant=False
if [ ${trainer} = ${pact_key} ]; then
run_train=${pact_trainer}
......@@ -344,6 +343,7 @@ else
# run eval
if [ ${eval_py} != "null" ]; then
eval ${env}
set_eval_params1=$(func_set_params "${eval_key1}" "${eval_value1}")
eval_cmd="${python} ${eval_py} ${set_eval_pretrain} ${set_use_gpu} ${set_eval_params1}"
eval $eval_cmd
......
tools/infer/predict_system.py
View file @ cf613b13
......@@ -24,6 +24,7 @@ os.environ["FLAGS_allocator_strategy"] = 'auto_growth'
import cv2
import copy
import numpy as np
import json
import time
import logging
from PIL import Image
......@@ -92,11 +93,11 @@ class TextSystem(object):
self.draw_crop_rec_res(self.args.crop_res_save_dir, img_crop_list,
rec_res)
filter_boxes, filter_rec_res = [], []
for box, rec_reuslt in zip(dt_boxes, rec_res):
text, score = rec_reuslt
for box, rec_result in zip(dt_boxes, rec_res):
text, score = rec_result
if score >= self.drop_score:
filter_boxes.append(box)
filter_rec_res.append(rec_reuslt)
filter_rec_res.append(rec_result)
return filter_boxes, filter_rec_res
......@@ -128,6 +129,9 @@ def main(args):
is_visualize = True
font_path = args.vis_font_path
drop_score = args.drop_score
draw_img_save_dir = args.draw_img_save_dir
os.makedirs(draw_img_save_dir, exist_ok=True)
save_results = []
# warm up 10 times
if args.warmup:
......@@ -157,6 +161,14 @@ def main(args):
for text, score in rec_res:
logger.debug("{}, {:.3f}".format(text, score))
res = [{
"transcription": rec_res[idx][0],
"points": np.array(dt_boxes[idx]).astype(np.int32).tolist(),
} for idx in range(len(dt_boxes))]
save_pred = os.path.basename(image_file) + "\t" + json.dumps(
res, ensure_ascii=False) + "\n"
save_results.append(save_pred)
if is_visualize:
image = Image.fromarray(cv2.cvtColor(img, cv2.COLOR_BGR2RGB))
boxes = dt_boxes
......@@ -170,8 +182,6 @@ def main(args):
scores,
drop_score=drop_score,
font_path=font_path)
draw_img_save_dir = args.draw_img_save_dir
os.makedirs(draw_img_save_dir, exist_ok=True)
if flag:
image_file = image_file[:-3] + "png"
cv2.imwrite(
......@@ -185,6 +195,9 @@ def main(args):
text_sys.text_detector.autolog.report()
text_sys.text_recognizer.autolog.report()
with open(os.path.join(draw_img_save_dir, "system_results.txt"), 'w') as f:
f.writelines(save_results)
if __name__ == "__main__":
args = utility.parse_args()
......
tools/infer_cls.py
View file @ cf613b13
......@@ -73,8 +73,8 @@ def main():
images = paddle.to_tensor(images)
preds = model(images)
post_result = post_process_class(preds)
for rec_reuslt in post_result:
logger.info('\t result: {}'.format(rec_reuslt))
for rec_result in post_result:
logger.info('\t result: {}'.format(rec_result))
logger.info("success!")
......
tools/program.py
View file @ cf613b13
......@@ -21,7 +21,7 @@ import sys
import platform
import yaml
import time
import shutil
import datetime
import paddle
import paddle.distributed as dist
from tqdm import tqdm
......@@ -29,11 +29,10 @@ from argparse import ArgumentParser, RawDescriptionHelpFormatter
from ppocr.utils.stats import TrainingStats
from ppocr.utils.save_load import save_model
from ppocr.utils.utility import print_dict
from ppocr.utils.utility import print_dict, AverageMeter
from ppocr.utils.logging import get_logger
from ppocr.utils import profiler
from ppocr.data import build_dataloader
import numpy as np
class ArgsParser(ArgumentParser):
......@@ -48,7 +47,8 @@ class ArgsParser(ArgumentParser):
'--profiler_options',
type=str,
default=None,
help='The option of profiler, which should be in format \"key1=value1;key2=value2;key3=value3\".'
help='The option of profiler, which should be in format ' \
'\"key1=value1;key2=value2;key3=value3\".'
)
def parse_args(self, argv=None):
......@@ -99,7 +99,8 @@ def merge_config(config, opts):
sub_keys = key.split('.')
assert (
sub_keys[0] in config
), "the sub_keys can only be one of global_config: {}, but get: {}, please check your running command".format(
), "the sub_keys can only be one of global_config: {}, but get: " \
"{}, please check your running command".format(
config.keys(), sub_keys[0])
cur = config[sub_keys[0]]
for idx, sub_key in enumerate(sub_keys[1:]):
......@@ -145,6 +146,7 @@ def train(config,
scaler=None):
cal_metric_during_train = config['Global'].get('cal_metric_during_train',
False)
calc_epoch_interval = config['Global'].get('calc_epoch_interval', 1)
log_smooth_window = config['Global']['log_smooth_window']
epoch_num = config['Global']['epoch_num']
print_batch_step = config['Global']['print_batch_step']
......@@ -160,11 +162,13 @@ def train(config,
eval_batch_step = eval_batch_step[1]
if len(valid_dataloader) == 0:
logger.info(
'No Images in eval dataset, evaluation during training will be disabled'
'No Images in eval dataset, evaluation during training ' \
'will be disabled'
)
start_eval_step = 1e111
logger.info(
"During the training process, after the {}th iteration, an evaluation is run every {} iterations".
"During the training process, after the {}th iteration, " \
"an evaluation is run every {} iterations".
format(start_eval_step, eval_batch_step))
save_epoch_step = config['Global']['save_epoch_step']
save_model_dir = config['Global']['save_model_dir']
......@@ -189,10 +193,11 @@ def train(config,
start_epoch = best_model_dict[
'start_epoch'] if 'start_epoch' in best_model_dict else 1
train_reader_cost = 0.0
train_run_cost = 0.0
total_samples = 0
train_reader_cost = 0.0
train_batch_cost = 0.0
reader_start = time.time()
eta_meter = AverageMeter()
max_iter = len(train_dataloader) - 1 if platform.system(
) == "Windows" else len(train_dataloader)
......@@ -203,7 +208,6 @@ def train(config,
config, 'Train', device, logger, seed=epoch)
max_iter = len(train_dataloader) - 1 if platform.system(
) == "Windows" else len(train_dataloader)
for idx, batch in enumerate(train_dataloader):
profiler.add_profiler_step(profiler_options)
train_reader_cost += time.time() - reader_start
......@@ -214,7 +218,6 @@ def train(config,
if use_srn:
model_average = True
train_start = time.time()
# use amp
if scaler:
with paddle.amp.auto_cast():
......@@ -242,7 +245,19 @@ def train(config,
optimizer.step()
optimizer.clear_grad()
train_run_cost += time.time() - train_start
if cal_metric_during_train and epoch % calc_epoch_interval == 0: # only rec and cls need
batch = [item.numpy() for item in batch]
if model_type in ['table', 'kie']:
eval_class(preds, batch)
else:
post_result = post_process_class(preds, batch[1])
eval_class(post_result, batch)
metric = eval_class.get_metric()
train_stats.update(metric)
train_batch_time = time.time() - reader_start
train_batch_cost += train_batch_time
eta_meter.update(train_batch_time)
global_step += 1
total_samples += len(images)
......@@ -254,16 +269,6 @@ def train(config,
stats['lr'] = lr
train_stats.update(stats)
if cal_metric_during_train: # only rec and cls need
batch = [item.numpy() for item in batch]
if model_type in ['table', 'kie']:
eval_class(preds, batch)
else:
post_result = post_process_class(preds, batch[1])
eval_class(post_result, batch)
metric = eval_class.get_metric()
train_stats.update(metric)
if vdl_writer is not None and dist.get_rank() == 0:
for k, v in train_stats.get().items():
vdl_writer.add_scalar('TRAIN/{}'.format(k), v, global_step)
......@@ -273,19 +278,26 @@ def train(config,
(global_step > 0 and global_step % print_batch_step == 0) or
(idx >= len(train_dataloader) - 1)):
logs = train_stats.log()
strs = 'epoch: [{}/{}], global_step: {}, {}, avg_reader_cost: {:.5f} s, avg_batch_cost: {:.5f} s, avg_samples: {}, ips: {:.5f}'.format(
epoch, epoch_num, global_step, logs, train_reader_cost /
print_batch_step, (train_reader_cost + train_run_cost) /
print_batch_step, total_samples / print_batch_step,
total_samples / (train_reader_cost + train_run_cost))
eta_sec = ((epoch_num + 1 - epoch) * \
len(train_dataloader) - idx - 1) * eta_meter.avg
eta_sec_format = str(datetime.timedelta(seconds=int(eta_sec)))
strs = 'epoch: [{}/{}], global_step: {}, {}, avg_reader_cost: ' \
'{:.5f} s, avg_batch_cost: {:.5f} s, avg_samples: {}, ' \
'ips: {:.5f}, eta: {}'.format(
epoch, epoch_num, global_step, logs,
train_reader_cost / print_batch_step,
train_batch_cost / print_batch_step,
total_samples / print_batch_step,
total_samples / train_batch_cost, eta_sec_format)
logger.info(strs)
train_reader_cost = 0.0
train_run_cost = 0.0
total_samples = 0
train_reader_cost = 0.0
train_batch_cost = 0.0
# eval
if global_step > start_eval_step and \
(global_step - start_eval_step) % eval_batch_step == 0 and dist.get_rank() == 0:
(global_step - start_eval_step) % eval_batch_step == 0 \
and dist.get_rank() == 0:
if model_average:
Model_Average = paddle.incubate.optimizer.ModelAverage(
0.15,
......@@ -511,7 +523,7 @@ def preprocess(is_train=False):
config['Global']['distributed'] = dist.get_world_size() != 1
if config['Global']['use_visualdl']:
if config['Global']['use_visualdl'] and dist.get_rank() == 0:
from visualdl import LogWriter
save_model_dir = config['Global']['save_model_dir']
vdl_writer_path = '{}/vdl/'.format(save_model_dir)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment