Unverified Commit 9f56b219 authored by Yuting Jiang's avatar Yuting Jiang Committed by GitHub
Browse files

Benchmarks: Unify metric names of benchmarks (#252)

**Description**
Unify metric names of benchmarks.
parent c13ed2a2
......@@ -50,9 +50,8 @@ def test_pytorch_bert_base():
assert (benchmark.run_count == 1)
assert (benchmark.return_code == ReturnCode.SUCCESS)
for metric in [
'steptime_train_float32', 'throughput_train_float32', 'steptime_train_float16', 'throughput_train_float16',
'steptime_inference_float32', 'throughput_inference_float32', 'steptime_inference_float16',
'throughput_inference_float16'
'fp32_train_step_time', 'fp32_train_throughput', 'fp16_train_step_time', 'fp16_train_throughput',
'fp32_inference_step_time', 'fp32_inference_throughput', 'fp16_inference_step_time', 'fp16_inference_throughput'
]:
assert (len(benchmark.raw_data[metric]) == benchmark.run_count)
assert (len(benchmark.raw_data[metric][0]) == benchmark._args.num_steps)
......
......@@ -17,9 +17,9 @@ def test_pytorch_cnn_with_gpu():
parameters='--batch_size 1 --image_size 224 --num_classes 5 --num_warmup 2 --num_steps 4 \
--model_action train inference',
check_metrics=[
'steptime_train_float32', 'throughput_train_float32', 'steptime_train_float16', 'throughput_train_float16',
'steptime_inference_float32', 'throughput_inference_float32', 'steptime_inference_float16',
'throughput_inference_float16'
'fp32_train_step_time', 'fp32_train_throughput', 'fp16_train_step_time', 'fp16_train_throughput',
'fp32_inference_step_time', 'fp32_inference_throughput', 'fp16_inference_step_time',
'fp16_inference_throughput'
]
)
......@@ -32,8 +32,7 @@ def test_pytorch_cnn_no_gpu():
parameters='--batch_size 1 --image_size 224 --num_classes 5 --num_warmup 2 --num_steps 4 \
--model_action train inference --precision float32 --no_gpu',
check_metrics=[
'steptime_train_float32', 'throughput_train_float32', 'steptime_inference_float32',
'throughput_inference_float32'
'fp32_train_step_time', 'fp32_train_throughput', 'fp32_inference_step_time', 'fp32_inference_throughput'
]
)
......
......@@ -49,9 +49,8 @@ def test_pytorch_gpt2_small():
assert (benchmark.run_count == 1)
assert (benchmark.return_code == ReturnCode.SUCCESS)
for metric in [
'steptime_train_float32', 'throughput_train_float32', 'steptime_train_float16', 'throughput_train_float16',
'steptime_inference_float32', 'throughput_inference_float32', 'steptime_inference_float16',
'throughput_inference_float16'
'fp32_train_step_time', 'fp32_train_throughput', 'fp16_train_step_time', 'fp16_train_throughput',
'fp32_inference_step_time', 'fp32_inference_throughput', 'fp16_inference_step_time', 'fp16_inference_throughput'
]:
assert (len(benchmark.raw_data[metric]) == benchmark.run_count)
assert (len(benchmark.raw_data[metric][0]) == benchmark._args.num_steps)
......
......@@ -16,9 +16,9 @@ def test_pytorch_lstm_with_gpu():
parameters='--batch_size 1 --num_classes 5 --seq_len 8 --num_warmup 2 --num_steps 4 \
--model_action train inference',
check_metrics=[
'steptime_train_float32', 'throughput_train_float32', 'steptime_train_float16', 'throughput_train_float16',
'steptime_inference_float32', 'throughput_inference_float32', 'steptime_inference_float16',
'throughput_inference_float16'
'fp32_train_step_time', 'fp32_train_throughput', 'fp16_train_step_time', 'fp16_train_throughput',
'fp32_inference_step_time', 'fp32_inference_throughput', 'fp16_inference_step_time',
'fp16_inference_throughput'
]
)
......@@ -30,8 +30,7 @@ def test_pytorch_lstm_no_gpu():
parameters='--batch_size 1 --num_classes 5 --seq_len 8 --num_warmup 2 --num_steps 4 \
--model_action train inference --precision float32 --no_gpu',
check_metrics=[
'steptime_train_float32', 'throughput_train_float32', 'steptime_inference_float32',
'throughput_inference_float32'
'fp32_train_step_time', 'fp32_train_throughput', 'fp32_inference_step_time', 'fp32_inference_throughput'
]
)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment