_benchmark.py 4.77 KB
Newer Older
Hongkun Yu's avatar
Hongkun Yu committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Flags for benchmarking models."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from absl import flags

from official.utils.flags._conventions import help_wrap


40
41
def define_log_steps():
  flags.DEFINE_integer(
Hongkun Yu's avatar
Hongkun Yu committed
42
43
      name="log_steps",
      default=100,
44
45
46
47
48
      help="Frequency with which to log timing information with TimeHistory.")

  return []


49
50
51
52
53
54
55
56
57
58
59
60
61
def define_benchmark(benchmark_log_dir=True, bigquery_uploader=True):
  """Register benchmarking flags.

  Args:
    benchmark_log_dir: Create a flag to specify location for benchmark logging.
    bigquery_uploader: Create flags for uploading results to BigQuery.

  Returns:
    A list of flags for core.py to marks as key flags.
  """

  key_flags = []

62
  flags.DEFINE_enum(
Hongkun Yu's avatar
Hongkun Yu committed
63
64
      name="benchmark_logger_type",
      default="BaseBenchmarkLogger",
65
      enum_values=["BaseBenchmarkLogger", "BenchmarkFileLogger"],
66
67
68
      help=help_wrap("The type of benchmark logger to use. Defaults to using "
                     "BaseBenchmarkLogger which logs to STDOUT. Different "
                     "loggers will require other flags to be able to work."))
69
  flags.DEFINE_string(
Hongkun Yu's avatar
Hongkun Yu committed
70
71
72
      name="benchmark_test_id",
      short_name="bti",
      default=None,
73
74
75
76
77
78
      help=help_wrap("The unique test ID of the benchmark run. It could be the "
                     "combination of key parameters. It is hardware "
                     "independent and could be used compare the performance "
                     "between different test runs. This flag is designed for "
                     "human consumption, and does not have any impact within "
                     "the system."))
79

80
  define_log_steps()
81

82
83
  if benchmark_log_dir:
    flags.DEFINE_string(
Hongkun Yu's avatar
Hongkun Yu committed
84
85
86
87
        name="benchmark_log_dir",
        short_name="bld",
        default=None,
        help=help_wrap("The location of the benchmark logging."))
88
89
90

  if bigquery_uploader:
    flags.DEFINE_string(
Hongkun Yu's avatar
Hongkun Yu committed
91
92
93
        name="gcp_project",
        short_name="gp",
        default=None,
94
95
96
97
        help=help_wrap(
            "The GCP project name where the benchmark will be uploaded."))

    flags.DEFINE_string(
Hongkun Yu's avatar
Hongkun Yu committed
98
99
100
        name="bigquery_data_set",
        short_name="bds",
        default="test_benchmark",
101
102
103
104
        help=help_wrap(
            "The Bigquery dataset name where the benchmark will be uploaded."))

    flags.DEFINE_string(
Hongkun Yu's avatar
Hongkun Yu committed
105
106
107
        name="bigquery_run_table",
        short_name="brt",
        default="benchmark_run",
108
109
110
        help=help_wrap("The Bigquery table name where the benchmark run "
                       "information will be uploaded."))

111
    flags.DEFINE_string(
Hongkun Yu's avatar
Hongkun Yu committed
112
113
        name="bigquery_run_status_table",
        short_name="brst",
114
115
116
117
        default="benchmark_run_status",
        help=help_wrap("The Bigquery table name where the benchmark run "
                       "status information will be uploaded."))

118
    flags.DEFINE_string(
Hongkun Yu's avatar
Hongkun Yu committed
119
120
        name="bigquery_metric_table",
        short_name="bmt",
121
122
123
124
        default="benchmark_metric",
        help=help_wrap("The Bigquery table name where the benchmark metric "
                       "information will be uploaded."))

125
126
127
  @flags.multi_flags_validator(
      ["benchmark_logger_type", "benchmark_log_dir"],
      message="--benchmark_logger_type=BenchmarkFileLogger will require "
Hongkun Yu's avatar
Hongkun Yu committed
128
      "--benchmark_log_dir being set")
129
130
131
132
133
134
135
  def _check_benchmark_log_dir(flags_dict):
    benchmark_logger_type = flags_dict["benchmark_logger_type"]
    if benchmark_logger_type == "BenchmarkFileLogger":
      return flags_dict["benchmark_log_dir"]
    return True

  return key_flags