Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
tsoc
superbenchmark
Commits
9f56b219
Unverified
Commit
9f56b219
authored
Dec 09, 2021
by
Yuting Jiang
Committed by
GitHub
Dec 09, 2021
Browse files
Benchmarks: Unify metric names of benchmarks (#252)
**Description** Unify metric names of benchmarks.
parent
c13ed2a2
Changes
44
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
12 additions
and
16 deletions
+12
-16
tests/benchmarks/model_benchmarks/test_pytorch_bert.py
tests/benchmarks/model_benchmarks/test_pytorch_bert.py
+2
-3
tests/benchmarks/model_benchmarks/test_pytorch_cnn.py
tests/benchmarks/model_benchmarks/test_pytorch_cnn.py
+4
-5
tests/benchmarks/model_benchmarks/test_pytorch_gpt2.py
tests/benchmarks/model_benchmarks/test_pytorch_gpt2.py
+2
-3
tests/benchmarks/model_benchmarks/test_pytorch_lstm.py
tests/benchmarks/model_benchmarks/test_pytorch_lstm.py
+4
-5
No files found.
tests/benchmarks/model_benchmarks/test_pytorch_bert.py
View file @
9f56b219
...
...
@@ -50,9 +50,8 @@ def test_pytorch_bert_base():
assert
(
benchmark
.
run_count
==
1
)
assert
(
benchmark
.
return_code
==
ReturnCode
.
SUCCESS
)
for
metric
in
[
'steptime_train_float32'
,
'throughput_train_float32'
,
'steptime_train_float16'
,
'throughput_train_float16'
,
'steptime_inference_float32'
,
'throughput_inference_float32'
,
'steptime_inference_float16'
,
'throughput_inference_float16'
'fp32_train_step_time'
,
'fp32_train_throughput'
,
'fp16_train_step_time'
,
'fp16_train_throughput'
,
'fp32_inference_step_time'
,
'fp32_inference_throughput'
,
'fp16_inference_step_time'
,
'fp16_inference_throughput'
]:
assert
(
len
(
benchmark
.
raw_data
[
metric
])
==
benchmark
.
run_count
)
assert
(
len
(
benchmark
.
raw_data
[
metric
][
0
])
==
benchmark
.
_args
.
num_steps
)
...
...
tests/benchmarks/model_benchmarks/test_pytorch_cnn.py
View file @
9f56b219
...
...
@@ -17,9 +17,9 @@ def test_pytorch_cnn_with_gpu():
parameters
=
'--batch_size 1 --image_size 224 --num_classes 5 --num_warmup 2 --num_steps 4
\
--model_action train inference'
,
check_metrics
=
[
'steptime
_train_float32'
,
'throughput_train_float32'
,
'
steptime
_train_float16'
,
'throughput_train_float16
'
,
'steptime_inference_
float32'
,
'
throughput_inference_
float32'
,
'
steptime
_inference_float16
'
,
'
throughput
_inference_
float16
'
'
fp32_train_
step
_
time
'
,
'fp32_train_throughput'
,
'fp16_train_
step
_
time
'
,
'fp16_train_throughput
'
,
'
fp32_inference_
step
_
time
'
,
'fp32
_inference_throughput
'
,
'fp16
_inference_step
_
time'
,
'
fp16
_inference_
throughput
'
]
)
...
...
@@ -32,8 +32,7 @@ def test_pytorch_cnn_no_gpu():
parameters
=
'--batch_size 1 --image_size 224 --num_classes 5 --num_warmup 2 --num_steps 4
\
--model_action train inference --precision float32 --no_gpu'
,
check_metrics
=
[
'steptime_train_float32'
,
'throughput_train_float32'
,
'steptime_inference_float32'
,
'throughput_inference_float32'
'fp32_train_step_time'
,
'fp32_train_throughput'
,
'fp32_inference_step_time'
,
'fp32_inference_throughput'
]
)
...
...
tests/benchmarks/model_benchmarks/test_pytorch_gpt2.py
View file @
9f56b219
...
...
@@ -49,9 +49,8 @@ def test_pytorch_gpt2_small():
assert
(
benchmark
.
run_count
==
1
)
assert
(
benchmark
.
return_code
==
ReturnCode
.
SUCCESS
)
for
metric
in
[
'steptime_train_float32'
,
'throughput_train_float32'
,
'steptime_train_float16'
,
'throughput_train_float16'
,
'steptime_inference_float32'
,
'throughput_inference_float32'
,
'steptime_inference_float16'
,
'throughput_inference_float16'
'fp32_train_step_time'
,
'fp32_train_throughput'
,
'fp16_train_step_time'
,
'fp16_train_throughput'
,
'fp32_inference_step_time'
,
'fp32_inference_throughput'
,
'fp16_inference_step_time'
,
'fp16_inference_throughput'
]:
assert
(
len
(
benchmark
.
raw_data
[
metric
])
==
benchmark
.
run_count
)
assert
(
len
(
benchmark
.
raw_data
[
metric
][
0
])
==
benchmark
.
_args
.
num_steps
)
...
...
tests/benchmarks/model_benchmarks/test_pytorch_lstm.py
View file @
9f56b219
...
...
@@ -16,9 +16,9 @@ def test_pytorch_lstm_with_gpu():
parameters
=
'--batch_size 1 --num_classes 5 --seq_len 8 --num_warmup 2 --num_steps 4
\
--model_action train inference'
,
check_metrics
=
[
'steptime
_train_float32'
,
'throughput_train_float32'
,
'
steptime
_train_float16'
,
'throughput_train_float16
'
,
'steptime_inference_
float32'
,
'
throughput_inference_
float32'
,
'
steptime
_inference_float16
'
,
'
throughput
_inference_
float16
'
'
fp32_train_
step
_
time
'
,
'fp32_train_throughput'
,
'fp16_train_
step
_
time
'
,
'fp16_train_throughput
'
,
'
fp32_inference_
step
_
time
'
,
'fp32
_inference_throughput
'
,
'fp16
_inference_step
_
time'
,
'
fp16
_inference_
throughput
'
]
)
...
...
@@ -30,8 +30,7 @@ def test_pytorch_lstm_no_gpu():
parameters
=
'--batch_size 1 --num_classes 5 --seq_len 8 --num_warmup 2 --num_steps 4
\
--model_action train inference --precision float32 --no_gpu'
,
check_metrics
=
[
'steptime_train_float32'
,
'throughput_train_float32'
,
'steptime_inference_float32'
,
'throughput_inference_float32'
'fp32_train_step_time'
,
'fp32_train_throughput'
,
'fp32_inference_step_time'
,
'fp32_inference_throughput'
]
)
...
...
Prev
1
2
3
Next
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment