keras_imagenet_benchmark.py 11.3 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Executes Keras benchmarks and accuracy tests."""
from __future__ import print_function

import os
19
import time
20
21
22
23

from absl import flags

from official.resnet import imagenet_main
Toby Boyd's avatar
Toby Boyd committed
24
from official.resnet.keras import keras_benchmark
25
26
27
from official.resnet.keras import keras_common
from official.resnet.keras import keras_imagenet_main

Toby Boyd's avatar
Toby Boyd committed
28
29
MIN_TOP_1_ACCURACY = 0.76
MAX_TOP_1_ACCURACY = 0.77
30

Toby Boyd's avatar
Toby Boyd committed
31
FLAGS = flags.FLAGS
32
33


Toby Boyd's avatar
Toby Boyd committed
34
35
class Resnet50KerasAccuracy(keras_benchmark.KerasBenchmark):
  """Benchmark accuracy tests for ResNet50 in Keras."""
36

37
  def __init__(self, output_dir=None, root_data_dir=None, **kwargs):
38
39
40
41
42
    """A benchmark class.

    Args:
      output_dir: directory where to output e.g. log files
      root_data_dir: directory under which to look for dataset
43
44
45
      **kwargs: arbitrary named arguments. This is needed to make the
                constructor forward compatible in case PerfZero provides more
                named arguments before updating the constructor.
46
47
    """

48
49
50
    flag_methods = [
        keras_common.define_keras_flags, imagenet_main.define_imagenet_flags
    ]
Toby Boyd's avatar
Toby Boyd committed
51

52
    self.data_dir = os.path.join(root_data_dir, 'imagenet')
53
54
    super(Resnet50KerasAccuracy, self).__init__(
        output_dir=output_dir, flag_methods=flag_methods)
55

Toby Boyd's avatar
Toby Boyd committed
56
  def benchmark_graph_8_gpu(self):
57
58
    """Test Keras model with Keras fit/dist_strat and 8 GPUs."""
    self._setup()
Toby Boyd's avatar
Toby Boyd committed
59
    FLAGS.num_gpus = 8
60
    FLAGS.data_dir = self.data_dir
61
    FLAGS.batch_size = 128 * 8
Toby Boyd's avatar
Toby Boyd committed
62
    FLAGS.train_epochs = 90
63
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_8_gpu')
Toby Boyd's avatar
Toby Boyd committed
64
    FLAGS.dtype = 'fp32'
65
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
66
67

  def benchmark_8_gpu(self):
68
69
    """Test Keras model with eager, dist_strat and 8 GPUs."""
    self._setup()
Toby Boyd's avatar
Toby Boyd committed
70
    FLAGS.num_gpus = 8
71
    FLAGS.data_dir = self.data_dir
72
    FLAGS.batch_size = 128 * 8
Toby Boyd's avatar
Toby Boyd committed
73
    FLAGS.train_epochs = 90
74
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu')
Toby Boyd's avatar
Toby Boyd committed
75
76
    FLAGS.dtype = 'fp32'
    FLAGS.enable_eager = True
77
78
    # Add some thread tunings to improve performance.
    FLAGS.datasets_num_private_threads = 14
79
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
80

Reed's avatar
Reed committed
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
  def benchmark_8_gpu_fp16(self):
    """Test Keras model with eager, dist_strat, 8 GPUs, and fp16."""
    self._setup()
    FLAGS.num_gpus = 8
    FLAGS.data_dir = self.data_dir
    FLAGS.batch_size = 256 * 8
    FLAGS.train_epochs = 90
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_fp16')
    FLAGS.dtype = 'fp16'
    FLAGS.enable_eager = True
    self._run_and_report_benchmark()

  def benchmark_xla_8_gpu_fp16(self):
    """Test Keras model with XLA, eager, dist_strat, 8 GPUs and fp16."""
    self._setup()
    FLAGS.num_gpus = 8
    FLAGS.data_dir = self.data_dir
    FLAGS.batch_size = 256 * 8
    FLAGS.train_epochs = 90
    FLAGS.model_dir = self._get_model_dir('benchmark_xla_8_gpu_fp16')
    FLAGS.dtype = 'fp16'
    FLAGS.enable_eager = True
    FLAGS.enable_xla = True
    self._run_and_report_benchmark()

106
107
108
109
110
111
  def _run_and_report_benchmark(self):
    start_time_sec = time.time()
    stats = keras_imagenet_main.run(flags.FLAGS)
    wall_time_sec = time.time() - start_time_sec

    super(Resnet50KerasAccuracy, self)._report_benchmark(
Toby Boyd's avatar
Toby Boyd committed
112
        stats,
113
        wall_time_sec,
Toby Boyd's avatar
Toby Boyd committed
114
115
        top_1_min=MIN_TOP_1_ACCURACY,
        top_1_max=MAX_TOP_1_ACCURACY,
116
        total_batch_size=FLAGS.batch_size,
Toby Boyd's avatar
Toby Boyd committed
117
        log_steps=100)
118
119
120
121

  def _get_model_dir(self, folder_name):
    return os.path.join(self.output_dir, folder_name)

Toby Boyd's avatar
Toby Boyd committed
122
123
124
125
126

class Resnet50KerasBenchmarkBase(keras_benchmark.KerasBenchmark):
  """Resnet50 benchmarks."""

  def __init__(self, output_dir=None, default_flags=None):
127
128
129
    flag_methods = [
        keras_common.define_keras_flags, imagenet_main.define_imagenet_flags
    ]
Toby Boyd's avatar
Toby Boyd committed
130
131
132
133
134
135

    super(Resnet50KerasBenchmarkBase, self).__init__(
        output_dir=output_dir,
        flag_methods=flag_methods,
        default_flags=default_flags)

136
137
  def _run_and_report_benchmark(self):
    start_time_sec = time.time()
Toby Boyd's avatar
Toby Boyd committed
138
    stats = keras_imagenet_main.run(FLAGS)
139
140
141
142
143
144
145
    wall_time_sec = time.time() - start_time_sec

    super(Resnet50KerasBenchmarkBase, self)._report_benchmark(
        stats,
        wall_time_sec,
        total_batch_size=FLAGS.batch_size,
        log_steps=FLAGS.log_steps)
Toby Boyd's avatar
Toby Boyd committed
146
147

  def benchmark_1_gpu_no_dist_strat(self):
Haoyu Zhang's avatar
Haoyu Zhang committed
148
    """Test Keras model with 1 GPU, no distribution strategy."""
Toby Boyd's avatar
Toby Boyd committed
149
150
151
152
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = True
153
    FLAGS.distribution_strategy = 'off'
154
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_no_dist_strat')
Toby Boyd's avatar
Toby Boyd committed
155
    FLAGS.batch_size = 128
156
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
157
158

  def benchmark_graph_1_gpu_no_dist_strat(self):
Haoyu Zhang's avatar
Haoyu Zhang committed
159
    """Test Keras model in legacy graph mode with 1 GPU, no dist strat."""
Toby Boyd's avatar
Toby Boyd committed
160
161
162
163
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = False
164
    FLAGS.distribution_strategy = 'off'
165
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu_no_dist_strat')
Toby Boyd's avatar
Toby Boyd committed
166
    FLAGS.batch_size = 128
167
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
168
169

  def benchmark_1_gpu(self):
Haoyu Zhang's avatar
Haoyu Zhang committed
170
    """Test Keras model with 1 GPU."""
Toby Boyd's avatar
Toby Boyd committed
171
172
173
174
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = True
175
    FLAGS.distribution_strategy = 'default'
176
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu')
Toby Boyd's avatar
Toby Boyd committed
177
    FLAGS.batch_size = 128
178
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
179

Haoyu Zhang's avatar
Haoyu Zhang committed
180
181
182
183
184
185
186
187
188
189
190
191
  def benchmark_xla_1_gpu(self):
    """Test Keras model with XLA and 1 GPU."""
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = True
    FLAGS.enable_xla = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu')
    FLAGS.batch_size = 128
    self._run_and_report_benchmark()

Reed's avatar
Reed committed
192
  def benchmark_1_gpu_fp16(self):
193
    """Test Keras model with 1 GPU and fp16."""
Reed's avatar
Reed committed
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_fp16')
    FLAGS.dtype = 'fp16'
    FLAGS.batch_size = 256
    self._run_and_report_benchmark()

  def benchmark_xla_1_gpu_fp16(self):
    """Test Keras model with XLA, 1 GPU and fp16."""
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = True
    FLAGS.enable_xla = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_xla_1_gpu_fp16')
    FLAGS.dtype = 'fp16'
    FLAGS.batch_size = 256
    self._run_and_report_benchmark()

Toby Boyd's avatar
Toby Boyd committed
217
  def benchmark_graph_1_gpu(self):
Haoyu Zhang's avatar
Haoyu Zhang committed
218
    """Test Keras model in legacy graph mode with 1 GPU."""
Toby Boyd's avatar
Toby Boyd committed
219
220
221
222
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = False
223
    FLAGS.distribution_strategy = 'default'
224
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu')
Toby Boyd's avatar
Toby Boyd committed
225
    FLAGS.batch_size = 128
226
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
227

Haoyu Zhang's avatar
Haoyu Zhang committed
228
229
230
231
232
233
234
235
236
237
238
239
  def benchmark_graph_xla_1_gpu(self):
    """Test Keras model in legacy graph mode with XLA and 1 GPU."""
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = False
    FLAGS.enable_xla = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_xla_1_gpu')
    FLAGS.batch_size = 128
    self._run_and_report_benchmark()

Toby Boyd's avatar
Toby Boyd committed
240
  def benchmark_8_gpu(self):
Haoyu Zhang's avatar
Haoyu Zhang committed
241
    """Test Keras model with 8 GPUs."""
Toby Boyd's avatar
Toby Boyd committed
242
243
244
245
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.enable_eager = True
246
    FLAGS.distribution_strategy = 'default'
247
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu')
Toby Boyd's avatar
Toby Boyd committed
248
    FLAGS.batch_size = 128 * 8  # 8 GPUs
249
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
250

251
  def benchmark_8_gpu_tweaked(self):
Haoyu Zhang's avatar
Haoyu Zhang committed
252
    """Test Keras model with manual config tuning and 8 GPUs."""
253
254
255
256
257
258
259
260
261
262
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.enable_eager = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_tweaked')
    FLAGS.batch_size = 128 * 8  # 8 GPUs
    FLAGS.datasets_num_private_threads = 14
    self._run_and_report_benchmark()

Haoyu Zhang's avatar
Haoyu Zhang committed
263
264
265
266
267
268
269
270
271
  def benchmark_xla_8_gpu(self):
    """Test Keras model with XLA and 8 GPUs."""
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.enable_eager = True
    FLAGS.enable_xla = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_xla_8_gpu')
272
    FLAGS.batch_size = 128 * 8  # 8 GPUs
Haoyu Zhang's avatar
Haoyu Zhang committed
273
274
    self._run_and_report_benchmark()

Reed's avatar
Reed committed
275
  def benchmark_8_gpu_fp16(self):
276
    """Test Keras model with 8 GPUs and fp16."""
Reed's avatar
Reed committed
277
278
279
280
281
282
283
284
285
286
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.enable_eager = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_fp16')
    FLAGS.batch_size = 256 * 8  # 8 GPUs
    self._run_and_report_benchmark()

  def benchmark_xla_8_gpu_fp16(self):
287
    """Test Keras model with XLA, 8 GPUs and fp16."""
Reed's avatar
Reed committed
288
289
290
291
292
293
294
295
296
297
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.enable_eager = True
    FLAGS.enable_xla = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_xla_8_gpu_fp16')
    FLAGS.batch_size = 256 * 8  # 8 GPUs
    self._run_and_report_benchmark()

Toby Boyd's avatar
Toby Boyd committed
298
  def benchmark_graph_8_gpu(self):
Haoyu Zhang's avatar
Haoyu Zhang committed
299
    """Test Keras model in legacy graph mode with 8 GPUs."""
Toby Boyd's avatar
Toby Boyd committed
300
301
302
303
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.enable_eager = False
304
    FLAGS.distribution_strategy = 'default'
305
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_8_gpu')
Toby Boyd's avatar
Toby Boyd committed
306
    FLAGS.batch_size = 128 * 8  # 8 GPUs
307
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
308

Haoyu Zhang's avatar
Haoyu Zhang committed
309
310
311
312
313
314
315
316
317
318
319
320
321
  def benchmark_graph_xla_8_gpu(self):
    """Test Keras model in legacy graph mode with XLA and 8 GPUs."""
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.enable_eager = False
    FLAGS.enable_xla = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_xla_8_gpu')
    # TODO(haoyuzhang): Set size to 128 per GPU when multi-GPU XLA OOM is fixed
    FLAGS.batch_size = 64 * 8  # 8 GPUs
    self._run_and_report_benchmark()

Toby Boyd's avatar
Toby Boyd committed
322
323
324
325
326
327
  def fill_report_object(self, stats):
    super(Resnet50KerasBenchmarkBase, self).fill_report_object(
        stats,
        total_batch_size=FLAGS.batch_size,
        log_steps=FLAGS.log_steps)

Toby Boyd's avatar
Toby Boyd committed
328
329
330
331

class Resnet50KerasBenchmarkSynth(Resnet50KerasBenchmarkBase):
  """Resnet50 synthetic benchmark tests."""

332
  def __init__(self, output_dir=None, root_data_dir=None, **kwargs):
Toby Boyd's avatar
Toby Boyd committed
333
334
335
336
337
338
    def_flags = {}
    def_flags['skip_eval'] = True
    def_flags['use_synthetic_data'] = True
    def_flags['train_steps'] = 110
    def_flags['log_steps'] = 10

339
340
    super(Resnet50KerasBenchmarkSynth, self).__init__(
        output_dir=output_dir, default_flags=def_flags)
Toby Boyd's avatar
Toby Boyd committed
341
342
343
344
345


class Resnet50KerasBenchmarkReal(Resnet50KerasBenchmarkBase):
  """Resnet50 real data benchmark tests."""

346
  def __init__(self, output_dir=None, root_data_dir=None, **kwargs):
Toby Boyd's avatar
Toby Boyd committed
347
348
    def_flags = {}
    def_flags['skip_eval'] = True
349
    def_flags['data_dir'] = os.path.join(root_data_dir, 'imagenet')
Toby Boyd's avatar
Toby Boyd committed
350
351
352
    def_flags['train_steps'] = 110
    def_flags['log_steps'] = 10

353
354
    super(Resnet50KerasBenchmarkReal, self).__init__(
        output_dir=output_dir, default_flags=def_flags)