keras_imagenet_benchmark.py 12.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Executes Keras benchmarks and accuracy tests."""
from __future__ import print_function

import os
19
import time
20
21
22
23

from absl import flags

from official.resnet import imagenet_main
Toby Boyd's avatar
Toby Boyd committed
24
from official.resnet.keras import keras_benchmark
25
26
27
from official.resnet.keras import keras_common
from official.resnet.keras import keras_imagenet_main

Toby Boyd's avatar
Toby Boyd committed
28
29
MIN_TOP_1_ACCURACY = 0.76
MAX_TOP_1_ACCURACY = 0.77
30

Toby Boyd's avatar
Toby Boyd committed
31
FLAGS = flags.FLAGS
32
33


Toby Boyd's avatar
Toby Boyd committed
34
35
class Resnet50KerasAccuracy(keras_benchmark.KerasBenchmark):
  """Benchmark accuracy tests for ResNet50 in Keras."""
36

37
  def __init__(self, output_dir=None, root_data_dir=None, **kwargs):
38
39
40
41
42
    """A benchmark class.

    Args:
      output_dir: directory where to output e.g. log files
      root_data_dir: directory under which to look for dataset
43
44
45
      **kwargs: arbitrary named arguments. This is needed to make the
                constructor forward compatible in case PerfZero provides more
                named arguments before updating the constructor.
46
47
    """

48
49
50
    flag_methods = [
        keras_common.define_keras_flags, imagenet_main.define_imagenet_flags
    ]
Toby Boyd's avatar
Toby Boyd committed
51

52
    self.data_dir = os.path.join(root_data_dir, 'imagenet')
53
54
    super(Resnet50KerasAccuracy, self).__init__(
        output_dir=output_dir, flag_methods=flag_methods)
55

Toby Boyd's avatar
Toby Boyd committed
56
  def benchmark_graph_8_gpu(self):
57
58
    """Test Keras model with Keras fit/dist_strat and 8 GPUs."""
    self._setup()
Toby Boyd's avatar
Toby Boyd committed
59
    FLAGS.num_gpus = 8
60
    FLAGS.data_dir = self.data_dir
61
    FLAGS.batch_size = 128 * 8
Toby Boyd's avatar
Toby Boyd committed
62
    FLAGS.train_epochs = 90
63
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_8_gpu')
Toby Boyd's avatar
Toby Boyd committed
64
    FLAGS.dtype = 'fp32'
65
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
66
67

  def benchmark_8_gpu(self):
68
69
    """Test Keras model with eager, dist_strat and 8 GPUs."""
    self._setup()
Toby Boyd's avatar
Toby Boyd committed
70
    FLAGS.num_gpus = 8
71
    FLAGS.data_dir = self.data_dir
72
    FLAGS.batch_size = 128 * 8
Toby Boyd's avatar
Toby Boyd committed
73
    FLAGS.train_epochs = 90
74
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu')
Toby Boyd's avatar
Toby Boyd committed
75
76
    FLAGS.dtype = 'fp32'
    FLAGS.enable_eager = True
77
78
    # Add some thread tunings to improve performance.
    FLAGS.datasets_num_private_threads = 14
79
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
80

Reed's avatar
Reed committed
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
  def benchmark_8_gpu_fp16(self):
    """Test Keras model with eager, dist_strat, 8 GPUs, and fp16."""
    self._setup()
    FLAGS.num_gpus = 8
    FLAGS.data_dir = self.data_dir
    FLAGS.batch_size = 256 * 8
    FLAGS.train_epochs = 90
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_fp16')
    FLAGS.dtype = 'fp16'
    FLAGS.enable_eager = True
    self._run_and_report_benchmark()

  def benchmark_xla_8_gpu_fp16(self):
    """Test Keras model with XLA, eager, dist_strat, 8 GPUs and fp16."""
    self._setup()
    FLAGS.num_gpus = 8
    FLAGS.data_dir = self.data_dir
    FLAGS.batch_size = 256 * 8
    FLAGS.train_epochs = 90
    FLAGS.model_dir = self._get_model_dir('benchmark_xla_8_gpu_fp16')
    FLAGS.dtype = 'fp16'
    FLAGS.enable_eager = True
    FLAGS.enable_xla = True
104
105
    # Thread tuning to improve performance.
    FLAGS.tf_gpu_thread_mode = 'gpu_private'
Reed's avatar
Reed committed
106
107
    self._run_and_report_benchmark()

108
109
110
111
112
113
  def _run_and_report_benchmark(self):
    start_time_sec = time.time()
    stats = keras_imagenet_main.run(flags.FLAGS)
    wall_time_sec = time.time() - start_time_sec

    super(Resnet50KerasAccuracy, self)._report_benchmark(
Toby Boyd's avatar
Toby Boyd committed
114
        stats,
115
        wall_time_sec,
Toby Boyd's avatar
Toby Boyd committed
116
117
        top_1_min=MIN_TOP_1_ACCURACY,
        top_1_max=MAX_TOP_1_ACCURACY,
118
        total_batch_size=FLAGS.batch_size,
Toby Boyd's avatar
Toby Boyd committed
119
        log_steps=100)
120
121
122
123

  def _get_model_dir(self, folder_name):
    return os.path.join(self.output_dir, folder_name)

Toby Boyd's avatar
Toby Boyd committed
124
125
126
127
128

class Resnet50KerasBenchmarkBase(keras_benchmark.KerasBenchmark):
  """Resnet50 benchmarks."""

  def __init__(self, output_dir=None, default_flags=None):
129
130
131
    flag_methods = [
        keras_common.define_keras_flags, imagenet_main.define_imagenet_flags
    ]
Toby Boyd's avatar
Toby Boyd committed
132
133
134
135
136
137

    super(Resnet50KerasBenchmarkBase, self).__init__(
        output_dir=output_dir,
        flag_methods=flag_methods,
        default_flags=default_flags)

138
139
  def _run_and_report_benchmark(self):
    start_time_sec = time.time()
Toby Boyd's avatar
Toby Boyd committed
140
    stats = keras_imagenet_main.run(FLAGS)
141
142
143
144
145
146
147
    wall_time_sec = time.time() - start_time_sec

    super(Resnet50KerasBenchmarkBase, self)._report_benchmark(
        stats,
        wall_time_sec,
        total_batch_size=FLAGS.batch_size,
        log_steps=FLAGS.log_steps)
Toby Boyd's avatar
Toby Boyd committed
148
149

  def benchmark_1_gpu_no_dist_strat(self):
Haoyu Zhang's avatar
Haoyu Zhang committed
150
    """Test Keras model with 1 GPU, no distribution strategy."""
Toby Boyd's avatar
Toby Boyd committed
151
152
153
154
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = True
155
    FLAGS.distribution_strategy = 'off'
156
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_no_dist_strat')
Toby Boyd's avatar
Toby Boyd committed
157
    FLAGS.batch_size = 128
158
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
159
160

  def benchmark_graph_1_gpu_no_dist_strat(self):
Haoyu Zhang's avatar
Haoyu Zhang committed
161
    """Test Keras model in legacy graph mode with 1 GPU, no dist strat."""
Toby Boyd's avatar
Toby Boyd committed
162
163
164
165
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = False
166
    FLAGS.distribution_strategy = 'off'
167
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu_no_dist_strat')
Toby Boyd's avatar
Toby Boyd committed
168
    FLAGS.batch_size = 128
169
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
170
171

  def benchmark_1_gpu(self):
Haoyu Zhang's avatar
Haoyu Zhang committed
172
    """Test Keras model with 1 GPU."""
Toby Boyd's avatar
Toby Boyd committed
173
174
175
176
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = True
177
    FLAGS.distribution_strategy = 'default'
178
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu')
Toby Boyd's avatar
Toby Boyd committed
179
    FLAGS.batch_size = 128
180
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
181

Haoyu Zhang's avatar
Haoyu Zhang committed
182
183
184
185
186
187
188
189
190
191
192
193
  def benchmark_xla_1_gpu(self):
    """Test Keras model with XLA and 1 GPU."""
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = True
    FLAGS.enable_xla = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu')
    FLAGS.batch_size = 128
    self._run_and_report_benchmark()

Reed's avatar
Reed committed
194
  def benchmark_1_gpu_fp16(self):
195
    """Test Keras model with 1 GPU and fp16."""
Reed's avatar
Reed committed
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_fp16')
    FLAGS.dtype = 'fp16'
    FLAGS.batch_size = 256
    self._run_and_report_benchmark()

  def benchmark_xla_1_gpu_fp16(self):
    """Test Keras model with XLA, 1 GPU and fp16."""
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = True
    FLAGS.enable_xla = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu_fp16')
    FLAGS.dtype = 'fp16'
    FLAGS.batch_size = 256
    self._run_and_report_benchmark()

Toby Boyd's avatar
Toby Boyd committed
219
  def benchmark_graph_1_gpu(self):
Haoyu Zhang's avatar
Haoyu Zhang committed
220
    """Test Keras model in legacy graph mode with 1 GPU."""
Toby Boyd's avatar
Toby Boyd committed
221
222
223
224
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = False
225
    FLAGS.distribution_strategy = 'default'
226
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu')
Toby Boyd's avatar
Toby Boyd committed
227
    FLAGS.batch_size = 128
228
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
229

Haoyu Zhang's avatar
Haoyu Zhang committed
230
231
232
233
234
235
236
237
238
239
240
241
  def benchmark_graph_xla_1_gpu(self):
    """Test Keras model in legacy graph mode with XLA and 1 GPU."""
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = False
    FLAGS.enable_xla = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_xla_1_gpu')
    FLAGS.batch_size = 128
    self._run_and_report_benchmark()

Toby Boyd's avatar
Toby Boyd committed
242
  def benchmark_8_gpu(self):
Haoyu Zhang's avatar
Haoyu Zhang committed
243
    """Test Keras model with 8 GPUs."""
Toby Boyd's avatar
Toby Boyd committed
244
245
246
247
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.enable_eager = True
248
    FLAGS.distribution_strategy = 'default'
249
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu')
Toby Boyd's avatar
Toby Boyd committed
250
    FLAGS.batch_size = 128 * 8  # 8 GPUs
251
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
252

253
  def benchmark_8_gpu_tweaked(self):
Haoyu Zhang's avatar
Haoyu Zhang committed
254
    """Test Keras model with manual config tuning and 8 GPUs."""
255
256
257
258
259
260
261
262
263
264
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.enable_eager = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_tweaked')
    FLAGS.batch_size = 128 * 8  # 8 GPUs
    FLAGS.datasets_num_private_threads = 14
    self._run_and_report_benchmark()

Haoyu Zhang's avatar
Haoyu Zhang committed
265
266
267
268
269
270
271
272
273
  def benchmark_xla_8_gpu(self):
    """Test Keras model with XLA and 8 GPUs."""
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.enable_eager = True
    FLAGS.enable_xla = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_xla_8_gpu')
274
    FLAGS.batch_size = 128 * 8  # 8 GPUs
Haoyu Zhang's avatar
Haoyu Zhang committed
275
276
    self._run_and_report_benchmark()

Reed's avatar
Reed committed
277
  def benchmark_8_gpu_fp16(self):
278
    """Test Keras model with 8 GPUs and fp16."""
Reed's avatar
Reed committed
279
280
281
    self._setup()

    FLAGS.num_gpus = 8
282
    FLAGS.dtype = 'fp16'
Reed's avatar
Reed committed
283
284
285
286
287
288
289
    FLAGS.enable_eager = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_fp16')
    FLAGS.batch_size = 256 * 8  # 8 GPUs
    self._run_and_report_benchmark()

  def benchmark_xla_8_gpu_fp16(self):
290
    """Test Keras model with XLA, 8 GPUs and fp16."""
Reed's avatar
Reed committed
291
292
293
    self._setup()

    FLAGS.num_gpus = 8
294
    FLAGS.dtype = 'fp16'
Reed's avatar
Reed committed
295
296
297
298
299
300
301
    FLAGS.enable_eager = True
    FLAGS.enable_xla = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_xla_8_gpu_fp16')
    FLAGS.batch_size = 256 * 8  # 8 GPUs
    self._run_and_report_benchmark()

302
303
304
305
306
307
308
309
310
311
312
313
314
315
  def benchmark_xla_8_gpu_fp16_tweaked(self):
    """Test Keras model with manual config tuning, XLA, 8 GPUs and fp16."""
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.dtype = 'fp16'
    FLAGS.enable_eager = True
    FLAGS.enable_xla = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_xla_8_gpu_fp16_tweaked')
    FLAGS.batch_size = 256 * 8  # 8 GPUs
    FLAGS.tf_gpu_thread_mode = 'gpu_private'
    self._run_and_report_benchmark()

316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
  def benchmark_xla_8_gpu_fp16_tensorboard_tweaked(self):
    """Test to track Tensorboard performance overhead."""
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.dtype = 'fp16'
    FLAGS.enable_eager = True
    FLAGS.enable_xla = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir(
        'benchmark_xla_8_gpu_fp16_tensorboard_tweaked')
    FLAGS.batch_size = 256 * 8  # 8 GPUs
    FLAGS.tf_gpu_thread_mode = 'gpu_private'
    FLAGS.enable_tensorboard = True
    self._run_and_report_benchmark()

Toby Boyd's avatar
Toby Boyd committed
332
  def benchmark_graph_8_gpu(self):
Haoyu Zhang's avatar
Haoyu Zhang committed
333
    """Test Keras model in legacy graph mode with 8 GPUs."""
Toby Boyd's avatar
Toby Boyd committed
334
335
336
337
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.enable_eager = False
338
    FLAGS.distribution_strategy = 'default'
339
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_8_gpu')
Toby Boyd's avatar
Toby Boyd committed
340
    FLAGS.batch_size = 128 * 8  # 8 GPUs
341
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
342

Haoyu Zhang's avatar
Haoyu Zhang committed
343
344
345
346
347
348
349
350
351
  def benchmark_graph_xla_8_gpu(self):
    """Test Keras model in legacy graph mode with XLA and 8 GPUs."""
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.enable_eager = False
    FLAGS.enable_xla = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_xla_8_gpu')
352
    FLAGS.batch_size = 128 * 8  # 8 GPUs
Haoyu Zhang's avatar
Haoyu Zhang committed
353
354
    self._run_and_report_benchmark()

Toby Boyd's avatar
Toby Boyd committed
355
356
357
358
359
360
  def fill_report_object(self, stats):
    super(Resnet50KerasBenchmarkBase, self).fill_report_object(
        stats,
        total_batch_size=FLAGS.batch_size,
        log_steps=FLAGS.log_steps)

Toby Boyd's avatar
Toby Boyd committed
361
362
363
364

class Resnet50KerasBenchmarkSynth(Resnet50KerasBenchmarkBase):
  """Resnet50 synthetic benchmark tests."""

365
  def __init__(self, output_dir=None, root_data_dir=None, **kwargs):
Toby Boyd's avatar
Toby Boyd committed
366
367
368
369
370
371
    def_flags = {}
    def_flags['skip_eval'] = True
    def_flags['use_synthetic_data'] = True
    def_flags['train_steps'] = 110
    def_flags['log_steps'] = 10

372
373
    super(Resnet50KerasBenchmarkSynth, self).__init__(
        output_dir=output_dir, default_flags=def_flags)
Toby Boyd's avatar
Toby Boyd committed
374
375
376
377
378


class Resnet50KerasBenchmarkReal(Resnet50KerasBenchmarkBase):
  """Resnet50 real data benchmark tests."""

379
  def __init__(self, output_dir=None, root_data_dir=None, **kwargs):
Toby Boyd's avatar
Toby Boyd committed
380
381
    def_flags = {}
    def_flags['skip_eval'] = True
382
    def_flags['data_dir'] = os.path.join(root_data_dir, 'imagenet')
Toby Boyd's avatar
Toby Boyd committed
383
384
385
    def_flags['train_steps'] = 110
    def_flags['log_steps'] = 10

386
387
    super(Resnet50KerasBenchmarkReal, self).__init__(
        output_dir=output_dir, default_flags=def_flags)