Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
tsoc
superbenchmark
Commits
9f56b219
Unverified
Commit
9f56b219
authored
Dec 09, 2021
by
Yuting Jiang
Committed by
GitHub
Dec 09, 2021
Browse files
Benchmarks: Unify metric names of benchmarks (#252)
**Description** Unify metric names of benchmarks.
parent
c13ed2a2
Changes
44
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
12 additions
and
16 deletions
+12
-16
tests/benchmarks/model_benchmarks/test_pytorch_bert.py
tests/benchmarks/model_benchmarks/test_pytorch_bert.py
+2
-3
tests/benchmarks/model_benchmarks/test_pytorch_cnn.py
tests/benchmarks/model_benchmarks/test_pytorch_cnn.py
+4
-5
tests/benchmarks/model_benchmarks/test_pytorch_gpt2.py
tests/benchmarks/model_benchmarks/test_pytorch_gpt2.py
+2
-3
tests/benchmarks/model_benchmarks/test_pytorch_lstm.py
tests/benchmarks/model_benchmarks/test_pytorch_lstm.py
+4
-5
No files found.
tests/benchmarks/model_benchmarks/test_pytorch_bert.py
View file @
9f56b219
...
...
@@ -50,9 +50,8 @@ def test_pytorch_bert_base():
assert
(
benchmark
.
run_count
==
1
)
assert
(
benchmark
.
return_code
==
ReturnCode
.
SUCCESS
)
for
metric
in
[
'steptime_train_float32'
,
'throughput_train_float32'
,
'steptime_train_float16'
,
'throughput_train_float16'
,
'steptime_inference_float32'
,
'throughput_inference_float32'
,
'steptime_inference_float16'
,
'throughput_inference_float16'
'fp32_train_step_time'
,
'fp32_train_throughput'
,
'fp16_train_step_time'
,
'fp16_train_throughput'
,
'fp32_inference_step_time'
,
'fp32_inference_throughput'
,
'fp16_inference_step_time'
,
'fp16_inference_throughput'
]:
assert
(
len
(
benchmark
.
raw_data
[
metric
])
==
benchmark
.
run_count
)
assert
(
len
(
benchmark
.
raw_data
[
metric
][
0
])
==
benchmark
.
_args
.
num_steps
)
...
...
tests/benchmarks/model_benchmarks/test_pytorch_cnn.py
View file @
9f56b219
...
...
@@ -17,9 +17,9 @@ def test_pytorch_cnn_with_gpu():
parameters
=
'--batch_size 1 --image_size 224 --num_classes 5 --num_warmup 2 --num_steps 4
\
--model_action train inference'
,
check_metrics
=
[
'steptime
_train_float32'
,
'throughput_train_float32'
,
'
steptime
_train_float16'
,
'throughput_train_float16
'
,
'steptime_inference_
float32'
,
'
throughput_inference_
float32'
,
'
steptime
_inference_float16
'
,
'
throughput
_inference_
float16
'
'
fp32_train_
step
_
time
'
,
'fp32_train_throughput'
,
'fp16_train_
step
_
time
'
,
'fp16_train_throughput
'
,
'
fp32_inference_
step
_
time
'
,
'fp32
_inference_throughput
'
,
'fp16
_inference_step
_
time'
,
'
fp16
_inference_
throughput
'
]
)
...
...
@@ -32,8 +32,7 @@ def test_pytorch_cnn_no_gpu():
parameters
=
'--batch_size 1 --image_size 224 --num_classes 5 --num_warmup 2 --num_steps 4
\
--model_action train inference --precision float32 --no_gpu'
,
check_metrics
=
[
'steptime_train_float32'
,
'throughput_train_float32'
,
'steptime_inference_float32'
,
'throughput_inference_float32'
'fp32_train_step_time'
,
'fp32_train_throughput'
,
'fp32_inference_step_time'
,
'fp32_inference_throughput'
]
)
...
...
tests/benchmarks/model_benchmarks/test_pytorch_gpt2.py
View file @
9f56b219
...
...
@@ -49,9 +49,8 @@ def test_pytorch_gpt2_small():
assert
(
benchmark
.
run_count
==
1
)
assert
(
benchmark
.
return_code
==
ReturnCode
.
SUCCESS
)
for
metric
in
[
'steptime_train_float32'
,
'throughput_train_float32'
,
'steptime_train_float16'
,
'throughput_train_float16'
,
'steptime_inference_float32'
,
'throughput_inference_float32'
,
'steptime_inference_float16'
,
'throughput_inference_float16'
'fp32_train_step_time'
,
'fp32_train_throughput'
,
'fp16_train_step_time'
,
'fp16_train_throughput'
,
'fp32_inference_step_time'
,
'fp32_inference_throughput'
,
'fp16_inference_step_time'
,
'fp16_inference_throughput'
]:
assert
(
len
(
benchmark
.
raw_data
[
metric
])
==
benchmark
.
run_count
)
assert
(
len
(
benchmark
.
raw_data
[
metric
][
0
])
==
benchmark
.
_args
.
num_steps
)
...
...
tests/benchmarks/model_benchmarks/test_pytorch_lstm.py
View file @
9f56b219
...
...
@@ -16,9 +16,9 @@ def test_pytorch_lstm_with_gpu():
parameters
=
'--batch_size 1 --num_classes 5 --seq_len 8 --num_warmup 2 --num_steps 4
\
--model_action train inference'
,
check_metrics
=
[
'steptime
_train_float32'
,
'throughput_train_float32'
,
'
steptime
_train_float16'
,
'throughput_train_float16
'
,
'steptime_inference_
float32'
,
'
throughput_inference_
float32'
,
'
steptime
_inference_float16
'
,
'
throughput
_inference_
float16
'
'
fp32_train_
step
_
time
'
,
'fp32_train_throughput'
,
'fp16_train_
step
_
time
'
,
'fp16_train_throughput
'
,
'
fp32_inference_
step
_
time
'
,
'fp32
_inference_throughput
'
,
'fp16
_inference_step
_
time'
,
'
fp16
_inference_
throughput
'
]
)
...
...
@@ -30,8 +30,7 @@ def test_pytorch_lstm_no_gpu():
parameters
=
'--batch_size 1 --num_classes 5 --seq_len 8 --num_warmup 2 --num_steps 4
\
--model_action train inference --precision float32 --no_gpu'
,
check_metrics
=
[
'steptime_train_float32'
,
'throughput_train_float32'
,
'steptime_inference_float32'
,
'throughput_inference_float32'
'fp32_train_step_time'
,
'fp32_train_throughput'
,
'fp32_inference_step_time'
,
'fp32_inference_throughput'
]
)
...
...
Prev
1
2
3
Next
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment