keras_cifar_benchmark.py 12.6 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
Toby Boyd's avatar
Toby Boyd committed
15
"""Executes Keras benchmarks and accuracy tests."""
Shining Sun's avatar
Shining Sun committed
16
17
from __future__ import absolute_import
from __future__ import division
Toby Boyd's avatar
Toby Boyd committed
18
19
from __future__ import print_function

20
import os
21
import time
Toby Boyd's avatar
Toby Boyd committed
22
from absl import flags
23
import tensorflow as tf  # pylint: disable=g-bad-import-order
Toby Boyd's avatar
Toby Boyd committed
24
25

from official.resnet import cifar10_main as cifar_main
Toby Boyd's avatar
Toby Boyd committed
26
from official.resnet.keras import keras_benchmark
27
28
29
from official.resnet.keras import keras_cifar_main
from official.resnet.keras import keras_common

30
31
MIN_TOP_1_ACCURACY = 0.925
MAX_TOP_1_ACCURACY = 0.938
Toby Boyd's avatar
Toby Boyd committed
32

Toby Boyd's avatar
Toby Boyd committed
33
FLAGS = flags.FLAGS
34
CIFAR_DATA_DIR_NAME = 'cifar-10-batches-bin'
Toby Boyd's avatar
Toby Boyd committed
35

36

Toby Boyd's avatar
Toby Boyd committed
37
38
class Resnet56KerasAccuracy(keras_benchmark.KerasBenchmark):
  """Accuracy tests for ResNet56 Keras CIFAR-10."""
39

40
  def __init__(self, output_dir=None, root_data_dir=None, **kwargs):
41
42
43
44
45
    """A benchmark class.

    Args:
      output_dir: directory where to output e.g. log files
      root_data_dir: directory under which to look for dataset
46
47
48
      **kwargs: arbitrary named arguments. This is needed to make the
                constructor forward compatible in case PerfZero provides more
                named arguments before updating the constructor.
49
50
    """

51
    self.data_dir = os.path.join(root_data_dir, CIFAR_DATA_DIR_NAME)
52
    flag_methods = [keras_cifar_main.define_cifar_flags]
Toby Boyd's avatar
Toby Boyd committed
53

54
55
    super(Resnet56KerasAccuracy, self).__init__(
        output_dir=output_dir, flag_methods=flag_methods)
Toby Boyd's avatar
Toby Boyd committed
56

Toby Boyd's avatar
Toby Boyd committed
57
  def benchmark_graph_1_gpu(self):
58
    """Test keras based model with Keras fit and distribution strategies."""
Toby Boyd's avatar
Toby Boyd committed
59
    self._setup()
Toby Boyd's avatar
Toby Boyd committed
60
    FLAGS.num_gpus = 1
61
    FLAGS.data_dir = self.data_dir
Toby Boyd's avatar
Toby Boyd committed
62
63
    FLAGS.batch_size = 128
    FLAGS.train_epochs = 182
64
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu')
Toby Boyd's avatar
Toby Boyd committed
65
    FLAGS.dtype = 'fp32'
66
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
67
68

  def benchmark_1_gpu(self):
69
70
    """Test keras based model with eager and distribution strategies."""
    self._setup()
Toby Boyd's avatar
Toby Boyd committed
71
    FLAGS.num_gpus = 1
72
    FLAGS.data_dir = self.data_dir
Toby Boyd's avatar
Toby Boyd committed
73
74
    FLAGS.batch_size = 128
    FLAGS.train_epochs = 182
75
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu')
Toby Boyd's avatar
Toby Boyd committed
76
77
    FLAGS.dtype = 'fp32'
    FLAGS.enable_eager = True
78
    self._run_and_report_benchmark()
79

80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
  def benchmark_cpu(self):
    """Test keras based model on CPU."""
    self._setup()
    FLAGS.num_gpus = 0
    FLAGS.data_dir = self.data_dir
    FLAGS.batch_size = 128
    FLAGS.train_epochs = 182
    FLAGS.model_dir = self._get_model_dir('benchmark_cpu')
    FLAGS.dtype = 'fp32'
    FLAGS.enable_eager = True
    FLAGS.data_format = 'channels_last'
    self._run_and_report_benchmark()

  def benchmark_cpu_no_dist_strat(self):
    """Test keras based model on CPU without distribution strategies."""
    self._setup()
    FLAGS.num_gpus = 0
    FLAGS.data_dir = self.data_dir
    FLAGS.batch_size = 128
    FLAGS.train_epochs = 182
    FLAGS.model_dir = self._get_model_dir('benchmark_cpu_no_dist_strat')
    FLAGS.dtype = 'fp32'
    FLAGS.enable_eager = True
    FLAGS.distribution_strategy = 'off'
    FLAGS.data_format = 'channels_last'
    self._run_and_report_benchmark()

  def benchmark_cpu_no_dist_strat_run_eagerly(self):
    """Test keras based model on CPU w/forced eager and no dist_strat."""
    self._setup()
    FLAGS.num_gpus = 0
    FLAGS.data_dir = self.data_dir
    FLAGS.batch_size = 128
    FLAGS.train_epochs = 182
    FLAGS.model_dir = self._get_model_dir(
        'benchmark_cpu_no_dist_strat_run_eagerly')
    FLAGS.dtype = 'fp32'
    FLAGS.enable_eager = True
    FLAGS.run_eagerly = True
    FLAGS.distribution_strategy = 'off'
    FLAGS.data_format = 'channels_last'
    self._run_and_report_benchmark()

123
124
125
126
  def benchmark_1_gpu_no_dist_strat(self):
    """Test keras based model with eager and no dist strat."""
    self._setup()
    FLAGS.num_gpus = 1
127
    FLAGS.explicit_gpu_placement = True
128
129
130
131
132
133
134
135
136
    FLAGS.data_dir = self.data_dir
    FLAGS.batch_size = 128
    FLAGS.train_epochs = 182
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_no_dist_strat')
    FLAGS.dtype = 'fp32'
    FLAGS.enable_eager = True
    FLAGS.distribution_strategy = 'off'
    self._run_and_report_benchmark()

137
  def benchmark_1_gpu_no_dist_strat_run_eagerly(self):
138
    """Test keras based model w/forced eager and no dist_strat."""
139
140
141
142
143
144
145
146
147
148
149
150
151
    self._setup()
    FLAGS.num_gpus = 1
    FLAGS.data_dir = self.data_dir
    FLAGS.batch_size = 128
    FLAGS.train_epochs = 182
    FLAGS.model_dir = self._get_model_dir(
        'benchmark_1_gpu_no_dist_strat_run_eagerly')
    FLAGS.dtype = 'fp32'
    FLAGS.enable_eager = True
    FLAGS.run_eagerly = True
    FLAGS.distribution_strategy = 'off'
    self._run_and_report_benchmark()

Toby Boyd's avatar
Toby Boyd committed
152
  def benchmark_2_gpu(self):
153
154
    """Test keras based model with eager and distribution strategies."""
    self._setup()
Toby Boyd's avatar
Toby Boyd committed
155
    FLAGS.num_gpus = 2
156
    FLAGS.data_dir = self.data_dir
Toby Boyd's avatar
Toby Boyd committed
157
158
    FLAGS.batch_size = 128
    FLAGS.train_epochs = 182
159
    FLAGS.model_dir = self._get_model_dir('benchmark_2_gpu')
Toby Boyd's avatar
Toby Boyd committed
160
161
    FLAGS.dtype = 'fp32'
    FLAGS.enable_eager = True
162
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
163
164

  def benchmark_graph_2_gpu(self):
165
166
    """Test keras based model with Keras fit and distribution strategies."""
    self._setup()
Toby Boyd's avatar
Toby Boyd committed
167
    FLAGS.num_gpus = 2
168
    FLAGS.data_dir = self.data_dir
Toby Boyd's avatar
Toby Boyd committed
169
170
    FLAGS.batch_size = 128
    FLAGS.train_epochs = 182
171
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_2_gpu')
Toby Boyd's avatar
Toby Boyd committed
172
    FLAGS.dtype = 'fp32'
173
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
174
175

  def benchmark_graph_1_gpu_no_dist_strat(self):
176
    """Test keras based model with Keras fit but not distribution strategies."""
Toby Boyd's avatar
Toby Boyd committed
177
    self._setup()
178
    FLAGS.distribution_strategy = 'off'
Toby Boyd's avatar
Toby Boyd committed
179
    FLAGS.num_gpus = 1
180
    FLAGS.data_dir = self.data_dir
Toby Boyd's avatar
Toby Boyd committed
181
182
    FLAGS.batch_size = 128
    FLAGS.train_epochs = 182
183
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu_no_dist_strat')
Toby Boyd's avatar
Toby Boyd committed
184
    FLAGS.dtype = 'fp32'
185
186
187
188
    self._run_and_report_benchmark()

  def _run_and_report_benchmark(self):
    start_time_sec = time.time()
Toby Boyd's avatar
Toby Boyd committed
189
    stats = keras_cifar_main.run(FLAGS)
190
    wall_time_sec = time.time() - start_time_sec
Toby Boyd's avatar
Toby Boyd committed
191

192
    super(Resnet56KerasAccuracy, self)._report_benchmark(
Toby Boyd's avatar
Toby Boyd committed
193
        stats,
194
        wall_time_sec,
Toby Boyd's avatar
Toby Boyd committed
195
196
        top_1_min=MIN_TOP_1_ACCURACY,
        top_1_max=MAX_TOP_1_ACCURACY,
197
        total_batch_size=FLAGS.batch_size,
Toby Boyd's avatar
Toby Boyd committed
198
199
200
201
202
203
204
        log_steps=100)


class Resnet56KerasBenchmarkBase(keras_benchmark.KerasBenchmark):
  """Short performance tests for ResNet56 via Keras and CIFAR-10."""

  def __init__(self, output_dir=None, default_flags=None):
205
    flag_methods = [keras_cifar_main.define_cifar_flags]
Toby Boyd's avatar
Toby Boyd committed
206
207
208
209
210
211

    super(Resnet56KerasBenchmarkBase, self).__init__(
        output_dir=output_dir,
        flag_methods=flag_methods,
        default_flags=default_flags)

212
213
  def _run_and_report_benchmark(self):
    start_time_sec = time.time()
Toby Boyd's avatar
Toby Boyd committed
214
    stats = keras_cifar_main.run(FLAGS)
215
216
217
218
219
220
221
    wall_time_sec = time.time() - start_time_sec

    super(Resnet56KerasBenchmarkBase, self)._report_benchmark(
        stats,
        wall_time_sec,
        total_batch_size=FLAGS.batch_size,
        log_steps=FLAGS.log_steps)
Toby Boyd's avatar
Toby Boyd committed
222

223
224
  def benchmark_1_gpu(self):
    """Test 1 gpu."""
225
226
227
    self._setup()
    FLAGS.num_gpus = 1
    FLAGS.enable_eager = True
228
229
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu')
230
231
232
    FLAGS.batch_size = 128
    self._run_and_report_benchmark()

233
234
  def benchmark_graph_1_gpu(self):
    """Test 1 gpu graph."""
Toby Boyd's avatar
Toby Boyd committed
235
236
237
    self._setup()
    FLAGS.num_gpus = 1
    FLAGS.enable_eager = False
238
239
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu')
Toby Boyd's avatar
Toby Boyd committed
240
    FLAGS.batch_size = 128
241
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
242

243
244
  def benchmark_1_gpu_no_dist_strat(self):
    """Test 1 gpu without distribution strategies."""
Toby Boyd's avatar
Toby Boyd committed
245
246
247
    self._setup()
    FLAGS.num_gpus = 1
    FLAGS.enable_eager = True
248
249
    FLAGS.distribution_strategy = 'off'
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_no_dist_strat')
Toby Boyd's avatar
Toby Boyd committed
250
    FLAGS.batch_size = 128
251
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
252

253
254
  def benchmark_graph_1_gpu_no_dist_strat(self):
    """Test 1 gpu graph mode without distribution strategies."""
Toby Boyd's avatar
Toby Boyd committed
255
256
257
    self._setup()
    FLAGS.num_gpus = 1
    FLAGS.enable_eager = False
258
259
    FLAGS.distribution_strategy = 'off'
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu_no_dist_strat')
Toby Boyd's avatar
Toby Boyd committed
260
    FLAGS.batch_size = 128
261
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
262

263
  def benchmark_1_gpu_no_dist_strat_run_eagerly(self):
264
    """Test 1 gpu without distribution strategy and forced eager."""
265
266
267
268
269
270
271
272
273
274
275
    self._setup()
    FLAGS.num_gpus = 1
    FLAGS.batch_size = 128
    FLAGS.model_dir = self._get_model_dir(
        'benchmark_1_gpu_no_dist_strat_run_eagerly')
    FLAGS.dtype = 'fp32'
    FLAGS.enable_eager = True
    FLAGS.run_eagerly = True
    FLAGS.distribution_strategy = 'off'
    self._run_and_report_benchmark()

Toby Boyd's avatar
Toby Boyd committed
276
  def benchmark_2_gpu(self):
277
    """Test 2 gpu."""
Toby Boyd's avatar
Toby Boyd committed
278
279
280
    self._setup()
    FLAGS.num_gpus = 2
    FLAGS.enable_eager = True
281
    FLAGS.distribution_strategy = 'default'
282
    FLAGS.model_dir = self._get_model_dir('benchmark_2_gpu')
Toby Boyd's avatar
Toby Boyd committed
283
    FLAGS.batch_size = 128 * 2  # 2 GPUs
284
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
285
286

  def benchmark_graph_2_gpu(self):
287
    """Test 2 gpu graph mode."""
Toby Boyd's avatar
Toby Boyd committed
288
289
290
    self._setup()
    FLAGS.num_gpus = 2
    FLAGS.enable_eager = False
291
    FLAGS.distribution_strategy = 'default'
292
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_2_gpu')
Toby Boyd's avatar
Toby Boyd committed
293
    FLAGS.batch_size = 128 * 2  # 2 GPUs
294
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
295

296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
  def benchmark_cpu(self):
    """Test cpu."""
    self._setup()
    FLAGS.num_gpus = 0
    FLAGS.enable_eager = True
    FLAGS.model_dir = self._get_model_dir('benchmark_cpu')
    FLAGS.batch_size = 128
    FLAGS.data_format = 'channels_last'
    self._run_and_report_benchmark()

  def benchmark_graph_cpu(self):
    """Test cpu graph mode."""
    self._setup()
    FLAGS.num_gpus = 0
    FLAGS.enable_eager = False
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_cpu')
    FLAGS.batch_size = 128
    FLAGS.data_format = 'channels_last'
    self._run_and_report_benchmark()

  def benchmark_cpu_no_dist_strat_run_eagerly(self):
    """Test cpu without distribution strategy and forced eager."""
    self._setup()
    FLAGS.num_gpus = 0
    FLAGS.distribution_strategy = 'off'
    FLAGS.enable_eager = True
    FLAGS.run_eagerly = True
    FLAGS.model_dir = self._get_model_dir(
        'benchmark_cpu_no_dist_strat_run_eagerly')
    FLAGS.batch_size = 128
    FLAGS.data_format = 'channels_last'
    self._run_and_report_benchmark()

  def benchmark_cpu_no_dist_strat(self):
    """Test cpu without distribution strategies."""
    self._setup()
    FLAGS.num_gpus = 0
    FLAGS.enable_eager = True
    FLAGS.distribution_strategy = 'off'
    FLAGS.model_dir = self._get_model_dir('benchmark_cpu_no_dist_strat')
    FLAGS.batch_size = 128
    FLAGS.data_format = 'channels_last'
    self._run_and_report_benchmark()

  def benchmark_graph_cpu_no_dist_strat(self):
    """Test cpu graph mode without distribution strategies."""
    self._setup()
    FLAGS.num_gpus = 0
    FLAGS.enable_eager = False
    FLAGS.distribution_strategy = 'off'
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_cpu_no_dist_strat')
    FLAGS.batch_size = 128
    FLAGS.data_format = 'channels_last'
    self._run_and_report_benchmark()

Toby Boyd's avatar
Toby Boyd committed
351
352
353
354

class Resnet56KerasBenchmarkSynth(Resnet56KerasBenchmarkBase):
  """Synthetic benchmarks for ResNet56 and Keras."""

355
  def __init__(self, output_dir=None, root_data_dir=None, **kwargs):
356
357
358
359
360
    default_flags = {}
    default_flags['skip_eval'] = True
    default_flags['use_synthetic_data'] = True
    default_flags['train_steps'] = 110
    default_flags['log_steps'] = 10
Toby Boyd's avatar
Toby Boyd committed
361

362
    super(Resnet56KerasBenchmarkSynth, self).__init__(
363
        output_dir=output_dir, default_flags=default_flags)
Toby Boyd's avatar
Toby Boyd committed
364
365
366
367
368


class Resnet56KerasBenchmarkReal(Resnet56KerasBenchmarkBase):
  """Real data benchmarks for ResNet56 and Keras."""

369
  def __init__(self, output_dir=None, root_data_dir=None, **kwargs):
370
371
372
373
374
    default_flags = {}
    default_flags['skip_eval'] = True
    default_flags['data_dir'] = os.path.join(root_data_dir, CIFAR_DATA_DIR_NAME)
    default_flags['train_steps'] = 110
    default_flags['log_steps'] = 10
Toby Boyd's avatar
Toby Boyd committed
375

376
    super(Resnet56KerasBenchmarkReal, self).__init__(
377
        output_dir=output_dir, default_flags=default_flags)
378
379
380
381


if __name__ == '__main__':
  tf.test.main()