Skip to content

GitLab

Unverified Commit bc1a61b9 authored by guoshzhao's avatar guoshzhao Committed by GitHub
Browse files

Benchmarks: Code Revision - Calculate average value by using statistics module. (#148) 

**Description**
Replace `sum(results) / len(results)` with `statistics.mean(results)`
parent e41b1f62
Hide whitespace changes
Inline Side-by-side
Showing with 22 additions and 18 deletions
superbench/benchmarks/micro_benchmarks/cublas_function.py
View file @ bc1a61b9
......@@ -6,6 +6,7 @@
import os
import json
import yaml
import statistics
from superbench.common.utils import logger
from superbench.benchmarks import Platform, BenchmarkRegistry, ReturnCode
......@@ -290,7 +291,7 @@ def _process_raw_result(self, cmd_idx, raw_output):
raw_data = raw_data.split(',')
raw_data.pop()
raw_data = [float(item) for item in raw_data]
self._result.add_result(metric, sum(raw_data) / len(raw_data))
self._result.add_result(metric, statistics.mean(raw_data))
self._result.add_raw_data(metric, raw_data)
if 'Error' in line:
error = True
......
superbench/benchmarks/micro_benchmarks/micro_base.py
View file @ bc1a61b9
......@@ -6,6 +6,7 @@
import os
import subprocess
import shutil
import statistics
from abc import abstractmethod
from superbench.common.utils import logger
......@@ -69,7 +70,8 @@ def _process_numeric_result(self, metric, result, reduce_type=None):
return False
self._result.add_raw_data(metric, result)
self._result.add_result(metric, sum(result) / len(result), reduce_type)
self._result.add_result(metric, statistics.mean(result), reduce_type)
return True
def print_env_info(self):
......
superbench/benchmarks/micro_benchmarks/sharding_matmul.py
View file @ bc1a61b9
......@@ -13,6 +13,7 @@
import os
import time
import statistics
# TODO - add mechanism to import torch as needed according to docker.
import torch
......@@ -260,8 +261,7 @@ def _benchmark(self):
logger.info(
'Matmul sharding - round: {0}, name: {1}, shape: ({2}, {3}) * ({3}, {4}), mode: {5}, cost: {6} ms'.
format(self._curr_run_index, self._name, M, K, N, mode,
sum(elapse_times) / len(elapse_times))
format(self._curr_run_index, self._name, M, K, N, mode, statistics.mean(elapse_times))
)
return True
......
superbench/benchmarks/model_benchmarks/model_base.py
View file @ bc1a61b9
......@@ -5,6 +5,7 @@
import math
import time
import statistics
from abc import abstractmethod
from superbench.common.utils import logger
......@@ -230,8 +231,7 @@ def __train(self, precision):
logger.info(
'Average train time - round: {}, model: {}, precision: {}, step time: {:.6f} ms.'.format(
self._curr_run_index, self._name, precision,
sum(step_times) / len(step_times)
self._curr_run_index, self._name, precision, statistics.mean(step_times)
)
)
......@@ -255,8 +255,7 @@ def __inference(self, precision):
logger.info(
'Average inference time - round: {}, model: {}, precision: {}, step time: {:.6f} ms.'.format(
self._curr_run_index, self._name, precision,
sum(step_times) / len(step_times)
self._curr_run_index, self._name, precision, statistics.mean(step_times)
)
)
......@@ -358,7 +357,7 @@ def __process_model_result(self, model_action, precision, step_times):
metric = 'steptime_{}_{}'.format(model_action, precision)
self._result.add_raw_data(metric, step_times)
avg = sum(step_times) / len(step_times)
avg = statistics.mean(step_times)
self._result.add_result(metric, avg)
# The unit of step time is millisecond, use it to calculate the throughput with the unit samples/sec.
......@@ -366,7 +365,7 @@ def __process_model_result(self, model_action, precision, step_times):
throughput = [millisecond_per_second / step_time * self._args.batch_size for step_time in step_times]
metric = 'throughput_{}_{}'.format(model_action, precision)
self._result.add_raw_data(metric, throughput)
avg = sum(throughput) / len(throughput)
avg = statistics.mean(throughput)
self._result.add_result(metric, avg)
return True
......
tests/benchmarks/model_benchmarks/test_model_base.py
View file @ bc1a61b9
......@@ -74,7 +74,7 @@ def _train_step(self, precision):
"""
duration = []
for i in range(self._args.num_steps):
duration.append(2)
duration.append(2.0)
return duration
def _inference_step(self, precision):
......@@ -89,7 +89,7 @@ def _inference_step(self, precision):
"""
duration = []
for i in range(self._args.num_steps):
duration.append(4)
duration.append(4.0)
return duration
def _cal_params_count(self):
......@@ -216,7 +216,8 @@ def test_train():
benchmark = create_benchmark()
expected_result = (
'{"name": "pytorch-fake-model", "type": "model", "run_count": 1, "return_code": 0, '
'"start_time": null, "end_time": null, "raw_data": {"steptime_train_float32": [[2, 2, 2, 2, 2, 2, 2, 2]], '
'"start_time": null, "end_time": null, "raw_data": {'
'"steptime_train_float32": [[2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0]], '
'"throughput_train_float32": [[16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0]]}, '
'"result": {"steptime_train_float32": [2.0], "throughput_train_float32": [16000.0]}, '
'"reduce": {"steptime_train_float32": null, "throughput_train_float32": null}}'
......@@ -241,7 +242,8 @@ def test_inference():
benchmark = create_benchmark()
expected_result = (
'{"name": "pytorch-fake-model", "type": "model", "run_count": 1, "return_code": 0, '
'"start_time": null, "end_time": null, "raw_data": {"steptime_inference_float16": [[4, 4, 4, 4, 4, 4, 4, 4]], '
'"start_time": null, "end_time": null, "raw_data": {'
'"steptime_inference_float16": [[4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0, 4.0]], '
'"throughput_inference_float16": [[8000.0, 8000.0, 8000.0, 8000.0, 8000.0, 8000.0, 8000.0, 8000.0]]}, '
'"result": {"steptime_inference_float16": [4.0], "throughput_inference_float16": [8000.0]}, '
'"reduce": {"steptime_inference_float16": null, "throughput_inference_float16": null}}'
......@@ -272,9 +274,9 @@ def test_benchmark():
assert (benchmark.run_count == 1)
assert (benchmark.return_code == ReturnCode.SUCCESS)
expected_raw_data = {
'steptime_train_float32': [[2, 2, 2, 2, 2, 2, 2, 2]],
'steptime_train_float32': [[2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0]],
'throughput_train_float32': [[16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0]],
'steptime_train_float16': [[2, 2, 2, 2, 2, 2, 2, 2]],
'steptime_train_float16': [[2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0]],
'throughput_train_float16': [[16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0]]
}
assert (benchmark.raw_data == expected_raw_data)
......@@ -288,9 +290,9 @@ def test_benchmark():
expected_serialized_result = (
'{"name": "pytorch-fake-model", "type": "model", "run_count": 1, "return_code": 0, "start_time": null, '
'"end_time": null, "raw_data": {"steptime_train_float32": [[2, 2, 2, 2, 2, 2, 2, 2]], '
'"end_time": null, "raw_data": {"steptime_train_float32": [[2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0]], '
'"throughput_train_float32": [[16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0]], '
'"steptime_train_float16": [[2, 2, 2, 2, 2, 2, 2, 2]], '
'"steptime_train_float16": [[2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0]], '
'"throughput_train_float16": [[16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0, 16000.0]]}, '
'"result": {"steptime_train_float32": [2.0], "throughput_train_float32": [16000.0], '
'"steptime_train_float16": [2.0], "throughput_train_float16": [16000.0]}, '
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment