resnet_ctl_imagenet_benchmark.py 13.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Executes CTL benchmarks and accuracy tests."""
from __future__ import print_function

import os
import time

# pylint: disable=g-bad-import-order
from absl import flags
import tensorflow as tf

Hongkun Yu's avatar
Hongkun Yu committed
25
from official.vision.image_classification import common
Hongkun Yu's avatar
Hongkun Yu committed
26
from official.vision.image_classification import resnet_ctl_imagenet_main
27
from official.utils.testing.perfzero_benchmark import PerfZeroBenchmark
28
from official.utils.testing import benchmark_wrappers
29
from official.utils.flags import core as flags_core
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70

MIN_TOP_1_ACCURACY = 0.76
MAX_TOP_1_ACCURACY = 0.77

FLAGS = flags.FLAGS


class CtlBenchmark(PerfZeroBenchmark):
  """Base benchmark class with methods to simplify testing."""

  def __init__(self, output_dir=None, default_flags=None, flag_methods=None):
    self.output_dir = output_dir
    self.default_flags = default_flags or {}
    self.flag_methods = flag_methods or {}
    super(CtlBenchmark, self).__init__(
        output_dir=self.output_dir,
        default_flags=self.default_flags,
        flag_methods=self.flag_methods)

  def _report_benchmark(self,
                        stats,
                        wall_time_sec,
                        top_1_max=None,
                        top_1_min=None,
                        total_batch_size=None,
                        log_steps=None,
                        warmup=1):
    """Report benchmark results by writing to local protobuf file.

    Args:
      stats: dict returned from keras models with known entries.
      wall_time_sec: the during of the benchmark execution in seconds
      top_1_max: highest passing level for top_1 accuracy.
      top_1_min: lowest passing level for top_1 accuracy.
      total_batch_size: Global batch-size.
      log_steps: How often the log was created for stats['step_timestamp_log'].
      warmup: number of entries in stats['step_timestamp_log'] to ignore.
    """

    metrics = []
    if 'eval_acc' in stats:
71
72
73
74
75
76
77
78
79
80
81
82
83
      metrics.append({
          'name': 'accuracy_top_1',
          'value': stats['eval_acc'],
          'min_value': top_1_min,
          'max_value': top_1_max
      })
      metrics.append({'name': 'eval_loss', 'value': stats['eval_loss']})

      metrics.append({
          'name': 'top_1_train_accuracy',
          'value': stats['train_acc']
      })
      metrics.append({'name': 'train_loss', 'value': stats['train_loss']})
84
85

    if (warmup and 'step_timestamp_log' in stats and
Ruoxin Sang's avatar
Ruoxin Sang committed
86
87
        len(stats['step_timestamp_log']) > warmup + 1):
      # first entry in the time_log is start of step 0. The rest of the
88
89
      # entries are the end of each step recorded
      time_log = stats['step_timestamp_log']
Will Cromar's avatar
Will Cromar committed
90
91
92
      steps_elapsed = time_log[-1].batch_index - time_log[warmup].batch_index
      time_elapsed = time_log[-1].timestamp - time_log[warmup].timestamp
      examples_per_sec = total_batch_size * (steps_elapsed / time_elapsed)
93
      metrics.append({'name': 'exp_per_second', 'value': examples_per_sec})
94
95

    if 'avg_exp_per_second' in stats:
96
97
98
99
      metrics.append({
          'name': 'avg_exp_per_second',
          'value': stats['avg_exp_per_second']
      })
100

101
    flags_str = flags_core.get_nondefault_flags_as_str()
102
103
104
105
106
    self.report_benchmark(
        iters=-1,
        wall_time=wall_time_sec,
        metrics=metrics,
        extras={'flags': flags_str})
107
108
109
110
111
112
113
114
115
116
117
118


class Resnet50CtlAccuracy(CtlBenchmark):
  """Benchmark accuracy tests for ResNet50 in CTL."""

  def __init__(self, output_dir=None, root_data_dir=None, **kwargs):
    """A benchmark class.

    Args:
      output_dir: directory where to output e.g. log files
      root_data_dir: directory under which to look for dataset
      **kwargs: arbitrary named arguments. This is needed to make the
119
120
        constructor forward compatible in case PerfZero provides more named
        arguments before updating the constructor.
121
122
    """

Hongkun Yu's avatar
Hongkun Yu committed
123
    flag_methods = [common.define_keras_flags]
124

125
    self.data_dir = os.path.join(root_data_dir, 'imagenet')
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
    super(Resnet50CtlAccuracy, self).__init__(
        output_dir=output_dir, flag_methods=flag_methods)

  def benchmark_8_gpu(self):
    """Test Keras model with eager, dist_strat and 8 GPUs."""
    self._setup()
    FLAGS.num_gpus = 8
    FLAGS.data_dir = self.data_dir
    FLAGS.batch_size = 128 * 8
    FLAGS.train_epochs = 90
    FLAGS.epochs_between_evals = 10
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu')
    FLAGS.dtype = 'fp32'
    # Add some thread tunings to improve performance.
    FLAGS.datasets_num_private_threads = 14
    self._run_and_report_benchmark()

143
144
145
146
147
148
149
150
151
152
153
154
155
156
  def benchmark_8_gpu_fp16(self):
    """Test Keras model with eager, 8 GPUs with tf.keras mixed precision."""
    self._setup()
    FLAGS.num_gpus = 8
    FLAGS.data_dir = self.data_dir
    FLAGS.batch_size = 256 * 8
    FLAGS.train_epochs = 90
    FLAGS.epochs_between_evals = 10
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_fp16')
    FLAGS.dtype = 'fp16'
    # Add some thread tunings to improve performance.
    FLAGS.datasets_num_private_threads = 14
    self._run_and_report_benchmark()

157
  def benchmark_8_gpu_amp(self):
158
    """Test Keras model with 8 GPUs and mixed precision via graph rewrite."""
159
160
161
    self._setup()
    FLAGS.num_gpus = 8
    FLAGS.data_dir = self.data_dir
162
    FLAGS.batch_size = 256 * 8
163
164
165
166
167
168
169
170
171
    FLAGS.train_epochs = 90
    FLAGS.epochs_between_evals = 10
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_amp')
    FLAGS.dtype = 'fp16'
    FLAGS.fp16_implementation = 'graph_rewrite'
    # Add some thread tunings to improve performance.
    FLAGS.datasets_num_private_threads = 14
    self._run_and_report_benchmark()

172
  @benchmark_wrappers.enable_runtime_flags
173
174
  def _run_and_report_benchmark(self):
    start_time_sec = time.time()
Hongkun Yu's avatar
Hongkun Yu committed
175
    stats = resnet_ctl_imagenet_main.run(flags.FLAGS)
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
    wall_time_sec = time.time() - start_time_sec

    super(Resnet50CtlAccuracy, self)._report_benchmark(
        stats,
        wall_time_sec,
        top_1_min=MIN_TOP_1_ACCURACY,
        top_1_max=MAX_TOP_1_ACCURACY,
        total_batch_size=FLAGS.batch_size,
        log_steps=100)

  def _get_model_dir(self, folder_name):
    return os.path.join(self.output_dir, folder_name)


class Resnet50CtlBenchmarkBase(CtlBenchmark):
  """Resnet50 benchmarks."""

  def __init__(self, output_dir=None, default_flags=None):
Hongkun Yu's avatar
Hongkun Yu committed
194
    flag_methods = [common.define_keras_flags]
195
196
197
198
199
200

    super(Resnet50CtlBenchmarkBase, self).__init__(
        output_dir=output_dir,
        flag_methods=flag_methods,
        default_flags=default_flags)

201
  @benchmark_wrappers.enable_runtime_flags
202
203
  def _run_and_report_benchmark(self):
    start_time_sec = time.time()
Hongkun Yu's avatar
Hongkun Yu committed
204
    stats = resnet_ctl_imagenet_main.run(FLAGS)
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
    wall_time_sec = time.time() - start_time_sec

    # Number of logged step time entries that are excluded in performance
    # report. We keep results from last 100 batches in this case.
    warmup = (FLAGS.train_steps - 100) // FLAGS.log_steps

    super(Resnet50CtlBenchmarkBase, self)._report_benchmark(
        stats,
        wall_time_sec,
        total_batch_size=FLAGS.batch_size,
        log_steps=FLAGS.log_steps,
        warmup=warmup)

  def benchmark_1_gpu_no_dist_strat(self):
    """Test Keras model with 1 GPU, no distribution strategy."""
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.distribution_strategy = 'off'
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_no_dist_strat')
    FLAGS.batch_size = 128
    self._run_and_report_benchmark()

  def benchmark_1_gpu(self):
    """Test Keras model with 1 GPU."""
    self._setup()

    FLAGS.num_gpus = 1
233
    FLAGS.distribution_strategy = 'one_device'
234
235
236
237
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu')
    FLAGS.batch_size = 128
    self._run_and_report_benchmark()

238
239
240
241
242
243
244
245
246
247
248
  def benchmark_1_gpu_fp16(self):
    """Test Keras model with 1 GPU with tf.keras mixed precision."""
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.distribution_strategy = 'one_device'
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_fp16')
    FLAGS.batch_size = 256
    FLAGS.dtype = 'fp16'
    self._run_and_report_benchmark()

249
250
251
252
253
  def benchmark_1_gpu_amp(self):
    """Test Keras model with 1 GPU with automatic mixed precision."""
    self._setup()

    FLAGS.num_gpus = 1
254
    FLAGS.distribution_strategy = 'one_device'
255
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_amp')
256
    FLAGS.batch_size = 256
257
258
259
260
261
262
263
264
265
    FLAGS.dtype = 'fp16'
    FLAGS.fp16_implementation = 'graph_rewrite'
    self._run_and_report_benchmark()

  def benchmark_xla_1_gpu_amp(self):
    """Test Keras model with XLA and 1 GPU with automatic mixed precision."""
    self._setup()

    FLAGS.num_gpus = 1
266
    FLAGS.distribution_strategy = 'one_device'
267
    FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu_amp')
268
    FLAGS.batch_size = 256
269
270
271
272
273
    FLAGS.dtype = 'fp16'
    FLAGS.fp16_implementation = 'graph_rewrite'
    FLAGS.enable_xla = True
    self._run_and_report_benchmark()

274
275
276
277
278
  def benchmark_1_gpu_eager(self):
    """Test Keras model with 1 GPU in pure eager mode."""
    self._setup()

    FLAGS.num_gpus = 1
279
    FLAGS.distribution_strategy = 'one_device'
280
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_eager')
281
    FLAGS.batch_size = 128
282
    FLAGS.use_tf_function = False
283
    FLAGS.use_tf_while_loop = False
284
    FLAGS.single_l2_loss_op = True
285
286
    self._run_and_report_benchmark()

287
288
289
290
291
292
293
  def benchmark_1_gpu_fp16_eager(self):
    """Test Keras model with 1 GPU with fp16 and pure eager mode."""
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.distribution_strategy = 'one_device'
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_fp16_eager')
294
    FLAGS.batch_size = 250
295
296
    FLAGS.dtype = 'fp16'
    FLAGS.use_tf_function = False
297
    FLAGS.use_tf_while_loop = False
298
299
300
    FLAGS.single_l2_loss_op = True
    self._run_and_report_benchmark()

301
302
303
304
305
  def benchmark_8_gpu(self):
    """Test Keras model with 8 GPUs."""
    self._setup()

    FLAGS.num_gpus = 8
306
    FLAGS.distribution_strategy = 'mirrored'
307
308
309
310
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu')
    FLAGS.batch_size = 128 * 8  # 8 GPUs
    self._run_and_report_benchmark()

311
312
313
314
315
316
317
318
319
320
321
  def benchmark_8_gpu_fp16(self):
    """Test Keras model with 8 GPUs with tf.keras mixed precision."""
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.distribution_strategy = 'mirrored'
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_fp16')
    FLAGS.batch_size = 256 * 8  # 8 GPUs
    FLAGS.dtype = 'fp16'
    self._run_and_report_benchmark()

Sai Ganesh Bandiatmakuri's avatar
Sai Ganesh Bandiatmakuri committed
322
323
324
325
326
327
  def benchmark_8_gpu_eager(self):
    """Test Keras model with 8 GPUs, eager, fp32."""
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.use_tf_function = False
328
    FLAGS.use_tf_while_loop = False
Sai Ganesh Bandiatmakuri's avatar
Sai Ganesh Bandiatmakuri committed
329
330
331
332
333
334
335
336
337
338
339
340
    FLAGS.distribution_strategy = 'mirrored'
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_eager')
    FLAGS.batch_size = 128
    self._run_and_report_benchmark()

  def benchmark_8_gpu_eager_fp16(self):
    """Test Keras model with 8 GPUs, eager, fp16."""
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.dtype = 'fp16'
    FLAGS.use_tf_function = False
341
    FLAGS.use_tf_while_loop = False
Sai Ganesh Bandiatmakuri's avatar
Sai Ganesh Bandiatmakuri committed
342
343
344
345
346
    FLAGS.distribution_strategy = 'mirrored'
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_eager_fp16')
    FLAGS.batch_size = 128
    self._run_and_report_benchmark()

347
348
349
350
351
  def benchmark_8_gpu_amp(self):
    """Test Keras model with 8 GPUs with automatic mixed precision."""
    self._setup()

    FLAGS.num_gpus = 8
352
    FLAGS.distribution_strategy = 'mirrored'
353
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_amp')
354
    FLAGS.batch_size = 256 * 8  # 8 GPUs
355
356
357
358
359
360
361
362
363
    FLAGS.dtype = 'fp16'
    FLAGS.fp16_implementation = 'graph_rewrite'
    self._run_and_report_benchmark()

  def benchmark_xla_8_gpu_amp(self):
    """Test Keras model with XLA and 8 GPUs with automatic mixed precision."""
    self._setup()

    FLAGS.num_gpus = 8
364
    FLAGS.distribution_strategy = 'mirrored'
365
    FLAGS.model_dir = self._get_model_dir('benchmark_xla_8_gpu_amp')
366
    FLAGS.batch_size = 256 * 8  # 8 GPUs
367
368
369
370
371
    FLAGS.dtype = 'fp16'
    FLAGS.fp16_implementation = 'graph_rewrite'
    FLAGS.enable_xla = True
    self._run_and_report_benchmark()

372
373
  def fill_report_object(self, stats):
    super(Resnet50CtlBenchmarkBase, self).fill_report_object(
374
        stats, total_batch_size=FLAGS.batch_size, log_steps=FLAGS.log_steps)
375
376
377
378
379
380
381
382
383
384


class Resnet50CtlBenchmarkSynth(Resnet50CtlBenchmarkBase):
  """Resnet50 synthetic benchmark tests."""

  def __init__(self, output_dir=None, root_data_dir=None, **kwargs):
    def_flags = {}
    def_flags['skip_eval'] = True
    def_flags['use_synthetic_data'] = True
    def_flags['train_steps'] = 110
Ruoxin Sang's avatar
Ruoxin Sang committed
385
    def_flags['steps_per_loop'] = 20
386
387
388
389
390
391
392
393
394
395
396
397
    def_flags['log_steps'] = 10

    super(Resnet50CtlBenchmarkSynth, self).__init__(
        output_dir=output_dir, default_flags=def_flags)


class Resnet50CtlBenchmarkReal(Resnet50CtlBenchmarkBase):
  """Resnet50 real data benchmark tests."""

  def __init__(self, output_dir=None, root_data_dir=None, **kwargs):
    def_flags = {}
    def_flags['skip_eval'] = True
Zongwei Zhou's avatar
Zongwei Zhou committed
398
    def_flags['data_dir'] = os.path.join(root_data_dir, 'imagenet')
399
    def_flags['train_steps'] = 110
Ruoxin Sang's avatar
Ruoxin Sang committed
400
    def_flags['steps_per_loop'] = 20
401
402
403
404
405
    def_flags['log_steps'] = 10

    super(Resnet50CtlBenchmarkReal, self).__init__(
        output_dir=output_dir, default_flags=def_flags)

406

407
408
if __name__ == '__main__':
  tf.test.main()