Unverified Commit 9f56b219 authored by Yuting Jiang's avatar Yuting Jiang Committed by GitHub
Browse files

Benchmarks: Unify metric names of benchmarks (#252)

**Description**
Unify metric names of benchmarks.
parent c13ed2a2
...@@ -50,9 +50,8 @@ def test_pytorch_bert_base(): ...@@ -50,9 +50,8 @@ def test_pytorch_bert_base():
assert (benchmark.run_count == 1) assert (benchmark.run_count == 1)
assert (benchmark.return_code == ReturnCode.SUCCESS) assert (benchmark.return_code == ReturnCode.SUCCESS)
for metric in [ for metric in [
'steptime_train_float32', 'throughput_train_float32', 'steptime_train_float16', 'throughput_train_float16', 'fp32_train_step_time', 'fp32_train_throughput', 'fp16_train_step_time', 'fp16_train_throughput',
'steptime_inference_float32', 'throughput_inference_float32', 'steptime_inference_float16', 'fp32_inference_step_time', 'fp32_inference_throughput', 'fp16_inference_step_time', 'fp16_inference_throughput'
'throughput_inference_float16'
]: ]:
assert (len(benchmark.raw_data[metric]) == benchmark.run_count) assert (len(benchmark.raw_data[metric]) == benchmark.run_count)
assert (len(benchmark.raw_data[metric][0]) == benchmark._args.num_steps) assert (len(benchmark.raw_data[metric][0]) == benchmark._args.num_steps)
......
...@@ -17,9 +17,9 @@ def test_pytorch_cnn_with_gpu(): ...@@ -17,9 +17,9 @@ def test_pytorch_cnn_with_gpu():
parameters='--batch_size 1 --image_size 224 --num_classes 5 --num_warmup 2 --num_steps 4 \ parameters='--batch_size 1 --image_size 224 --num_classes 5 --num_warmup 2 --num_steps 4 \
--model_action train inference', --model_action train inference',
check_metrics=[ check_metrics=[
'steptime_train_float32', 'throughput_train_float32', 'steptime_train_float16', 'throughput_train_float16', 'fp32_train_step_time', 'fp32_train_throughput', 'fp16_train_step_time', 'fp16_train_throughput',
'steptime_inference_float32', 'throughput_inference_float32', 'steptime_inference_float16', 'fp32_inference_step_time', 'fp32_inference_throughput', 'fp16_inference_step_time',
'throughput_inference_float16' 'fp16_inference_throughput'
] ]
) )
...@@ -32,8 +32,7 @@ def test_pytorch_cnn_no_gpu(): ...@@ -32,8 +32,7 @@ def test_pytorch_cnn_no_gpu():
parameters='--batch_size 1 --image_size 224 --num_classes 5 --num_warmup 2 --num_steps 4 \ parameters='--batch_size 1 --image_size 224 --num_classes 5 --num_warmup 2 --num_steps 4 \
--model_action train inference --precision float32 --no_gpu', --model_action train inference --precision float32 --no_gpu',
check_metrics=[ check_metrics=[
'steptime_train_float32', 'throughput_train_float32', 'steptime_inference_float32', 'fp32_train_step_time', 'fp32_train_throughput', 'fp32_inference_step_time', 'fp32_inference_throughput'
'throughput_inference_float32'
] ]
) )
......
...@@ -49,9 +49,8 @@ def test_pytorch_gpt2_small(): ...@@ -49,9 +49,8 @@ def test_pytorch_gpt2_small():
assert (benchmark.run_count == 1) assert (benchmark.run_count == 1)
assert (benchmark.return_code == ReturnCode.SUCCESS) assert (benchmark.return_code == ReturnCode.SUCCESS)
for metric in [ for metric in [
'steptime_train_float32', 'throughput_train_float32', 'steptime_train_float16', 'throughput_train_float16', 'fp32_train_step_time', 'fp32_train_throughput', 'fp16_train_step_time', 'fp16_train_throughput',
'steptime_inference_float32', 'throughput_inference_float32', 'steptime_inference_float16', 'fp32_inference_step_time', 'fp32_inference_throughput', 'fp16_inference_step_time', 'fp16_inference_throughput'
'throughput_inference_float16'
]: ]:
assert (len(benchmark.raw_data[metric]) == benchmark.run_count) assert (len(benchmark.raw_data[metric]) == benchmark.run_count)
assert (len(benchmark.raw_data[metric][0]) == benchmark._args.num_steps) assert (len(benchmark.raw_data[metric][0]) == benchmark._args.num_steps)
......
...@@ -16,9 +16,9 @@ def test_pytorch_lstm_with_gpu(): ...@@ -16,9 +16,9 @@ def test_pytorch_lstm_with_gpu():
parameters='--batch_size 1 --num_classes 5 --seq_len 8 --num_warmup 2 --num_steps 4 \ parameters='--batch_size 1 --num_classes 5 --seq_len 8 --num_warmup 2 --num_steps 4 \
--model_action train inference', --model_action train inference',
check_metrics=[ check_metrics=[
'steptime_train_float32', 'throughput_train_float32', 'steptime_train_float16', 'throughput_train_float16', 'fp32_train_step_time', 'fp32_train_throughput', 'fp16_train_step_time', 'fp16_train_throughput',
'steptime_inference_float32', 'throughput_inference_float32', 'steptime_inference_float16', 'fp32_inference_step_time', 'fp32_inference_throughput', 'fp16_inference_step_time',
'throughput_inference_float16' 'fp16_inference_throughput'
] ]
) )
...@@ -30,8 +30,7 @@ def test_pytorch_lstm_no_gpu(): ...@@ -30,8 +30,7 @@ def test_pytorch_lstm_no_gpu():
parameters='--batch_size 1 --num_classes 5 --seq_len 8 --num_warmup 2 --num_steps 4 \ parameters='--batch_size 1 --num_classes 5 --seq_len 8 --num_warmup 2 --num_steps 4 \
--model_action train inference --precision float32 --no_gpu', --model_action train inference --precision float32 --no_gpu',
check_metrics=[ check_metrics=[
'steptime_train_float32', 'throughput_train_float32', 'steptime_inference_float32', 'fp32_train_step_time', 'fp32_train_throughput', 'fp32_inference_step_time', 'fp32_inference_throughput'
'throughput_inference_float32'
] ]
) )
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment