keras_imagenet_benchmark.py 7.09 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Executes Keras benchmarks and accuracy tests."""
from __future__ import print_function

import os
19
import time
20
21
22
23

from absl import flags

from official.resnet import imagenet_main
Toby Boyd's avatar
Toby Boyd committed
24
from official.resnet.keras import keras_benchmark
25
26
27
from official.resnet.keras import keras_common
from official.resnet.keras import keras_imagenet_main

Toby Boyd's avatar
Toby Boyd committed
28
29
MIN_TOP_1_ACCURACY = 0.76
MAX_TOP_1_ACCURACY = 0.77
30

Toby Boyd's avatar
Toby Boyd committed
31
FLAGS = flags.FLAGS
32
33


Toby Boyd's avatar
Toby Boyd committed
34
35
class Resnet50KerasAccuracy(keras_benchmark.KerasBenchmark):
  """Benchmark accuracy tests for ResNet50 in Keras."""
36

37
  def __init__(self, output_dir=None, root_data_dir=None, **kwargs):
38
39
40
41
42
    """A benchmark class.

    Args:
      output_dir: directory where to output e.g. log files
      root_data_dir: directory under which to look for dataset
43
44
45
      **kwargs: arbitrary named arguments. This is needed to make the
                constructor forward compatible in case PerfZero provides more
                named arguments before updating the constructor.
46
47
    """

48
49
50
    flag_methods = [
        keras_common.define_keras_flags, imagenet_main.define_imagenet_flags
    ]
Toby Boyd's avatar
Toby Boyd committed
51

52
    self.data_dir = os.path.join(root_data_dir, 'imagenet')
53
54
    super(Resnet50KerasAccuracy, self).__init__(
        output_dir=output_dir, flag_methods=flag_methods)
55

Toby Boyd's avatar
Toby Boyd committed
56
  def benchmark_graph_8_gpu(self):
57
58
    """Test Keras model with Keras fit/dist_strat and 8 GPUs."""
    self._setup()
Toby Boyd's avatar
Toby Boyd committed
59
    FLAGS.num_gpus = 8
60
    FLAGS.data_dir = self.data_dir
61
    FLAGS.batch_size = 128 * 8
Toby Boyd's avatar
Toby Boyd committed
62
    FLAGS.train_epochs = 90
63
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_8_gpu')
Toby Boyd's avatar
Toby Boyd committed
64
    FLAGS.dtype = 'fp32'
65
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
66
67

  def benchmark_8_gpu(self):
68
69
    """Test Keras model with eager, dist_strat and 8 GPUs."""
    self._setup()
Toby Boyd's avatar
Toby Boyd committed
70
    FLAGS.num_gpus = 8
71
    FLAGS.data_dir = self.data_dir
72
    FLAGS.batch_size = 128 * 8
Toby Boyd's avatar
Toby Boyd committed
73
    FLAGS.train_epochs = 90
74
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu')
Toby Boyd's avatar
Toby Boyd committed
75
76
    FLAGS.dtype = 'fp32'
    FLAGS.enable_eager = True
77
78
    # Add some thread tunings to improve performance.
    FLAGS.datasets_num_private_threads = 14
79
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
80

81
82
83
84
85
86
  def _run_and_report_benchmark(self):
    start_time_sec = time.time()
    stats = keras_imagenet_main.run(flags.FLAGS)
    wall_time_sec = time.time() - start_time_sec

    super(Resnet50KerasAccuracy, self)._report_benchmark(
Toby Boyd's avatar
Toby Boyd committed
87
        stats,
88
        wall_time_sec,
Toby Boyd's avatar
Toby Boyd committed
89
90
        top_1_min=MIN_TOP_1_ACCURACY,
        top_1_max=MAX_TOP_1_ACCURACY,
91
        total_batch_size=FLAGS.batch_size,
Toby Boyd's avatar
Toby Boyd committed
92
        log_steps=100)
93
94
95
96

  def _get_model_dir(self, folder_name):
    return os.path.join(self.output_dir, folder_name)

Toby Boyd's avatar
Toby Boyd committed
97
98
99
100
101

class Resnet50KerasBenchmarkBase(keras_benchmark.KerasBenchmark):
  """Resnet50 benchmarks."""

  def __init__(self, output_dir=None, default_flags=None):
102
103
104
    flag_methods = [
        keras_common.define_keras_flags, imagenet_main.define_imagenet_flags
    ]
Toby Boyd's avatar
Toby Boyd committed
105
106
107
108
109
110

    super(Resnet50KerasBenchmarkBase, self).__init__(
        output_dir=output_dir,
        flag_methods=flag_methods,
        default_flags=default_flags)

111
112
  def _run_and_report_benchmark(self):
    start_time_sec = time.time()
Toby Boyd's avatar
Toby Boyd committed
113
    stats = keras_imagenet_main.run(FLAGS)
114
115
116
117
118
119
120
    wall_time_sec = time.time() - start_time_sec

    super(Resnet50KerasBenchmarkBase, self)._report_benchmark(
        stats,
        wall_time_sec,
        total_batch_size=FLAGS.batch_size,
        log_steps=FLAGS.log_steps)
Toby Boyd's avatar
Toby Boyd committed
121
122
123
124
125
126

  def benchmark_1_gpu_no_dist_strat(self):
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = True
127
    FLAGS.distribution_strategy = 'off'
128
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu_no_dist_strat')
Toby Boyd's avatar
Toby Boyd committed
129
    FLAGS.batch_size = 128
130
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
131
132
133
134
135
136

  def benchmark_graph_1_gpu_no_dist_strat(self):
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = False
137
    FLAGS.distribution_strategy = 'off'
138
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu_no_dist_strat')
Toby Boyd's avatar
Toby Boyd committed
139
    FLAGS.batch_size = 128
140
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
141
142
143
144
145
146

  def benchmark_1_gpu(self):
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = True
147
    FLAGS.distribution_strategy = 'default'
148
    FLAGS.model_dir = self._get_model_dir('benchmark_1_gpu')
Toby Boyd's avatar
Toby Boyd committed
149
    FLAGS.batch_size = 128
150
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
151
152
153
154
155
156

  def benchmark_graph_1_gpu(self):
    self._setup()

    FLAGS.num_gpus = 1
    FLAGS.enable_eager = False
157
    FLAGS.distribution_strategy = 'default'
158
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_1_gpu')
Toby Boyd's avatar
Toby Boyd committed
159
    FLAGS.batch_size = 128
160
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
161
162
163
164
165
166

  def benchmark_8_gpu(self):
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.enable_eager = True
167
    FLAGS.distribution_strategy = 'default'
168
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu')
Toby Boyd's avatar
Toby Boyd committed
169
    FLAGS.batch_size = 128 * 8  # 8 GPUs
170
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
171

172
173
174
175
176
177
178
179
180
181
182
  def benchmark_8_gpu_tweaked(self):
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.enable_eager = True
    FLAGS.distribution_strategy = 'default'
    FLAGS.model_dir = self._get_model_dir('benchmark_8_gpu_tweaked')
    FLAGS.batch_size = 128 * 8  # 8 GPUs
    FLAGS.datasets_num_private_threads = 14
    self._run_and_report_benchmark()

Toby Boyd's avatar
Toby Boyd committed
183
184
185
186
187
  def benchmark_graph_8_gpu(self):
    self._setup()

    FLAGS.num_gpus = 8
    FLAGS.enable_eager = False
188
    FLAGS.distribution_strategy = 'default'
189
    FLAGS.model_dir = self._get_model_dir('benchmark_graph_8_gpu')
Toby Boyd's avatar
Toby Boyd committed
190
    FLAGS.batch_size = 128 * 8  # 8 GPUs
191
    self._run_and_report_benchmark()
Toby Boyd's avatar
Toby Boyd committed
192

Toby Boyd's avatar
Toby Boyd committed
193
194
195
196
197
198
  def fill_report_object(self, stats):
    super(Resnet50KerasBenchmarkBase, self).fill_report_object(
        stats,
        total_batch_size=FLAGS.batch_size,
        log_steps=FLAGS.log_steps)

Toby Boyd's avatar
Toby Boyd committed
199
200
201
202

class Resnet50KerasBenchmarkSynth(Resnet50KerasBenchmarkBase):
  """Resnet50 synthetic benchmark tests."""

203
  def __init__(self, output_dir=None, root_data_dir=None, **kwargs):
Toby Boyd's avatar
Toby Boyd committed
204
205
206
207
208
209
    def_flags = {}
    def_flags['skip_eval'] = True
    def_flags['use_synthetic_data'] = True
    def_flags['train_steps'] = 110
    def_flags['log_steps'] = 10

210
211
    super(Resnet50KerasBenchmarkSynth, self).__init__(
        output_dir=output_dir, default_flags=def_flags)
Toby Boyd's avatar
Toby Boyd committed
212
213
214
215
216


class Resnet50KerasBenchmarkReal(Resnet50KerasBenchmarkBase):
  """Resnet50 real data benchmark tests."""

217
  def __init__(self, output_dir=None, root_data_dir=None, **kwargs):
Toby Boyd's avatar
Toby Boyd committed
218
219
    def_flags = {}
    def_flags['skip_eval'] = True
220
    def_flags['data_dir'] = os.path.join(root_data_dir, 'imagenet')
Toby Boyd's avatar
Toby Boyd committed
221
222
223
    def_flags['train_steps'] = 110
    def_flags['log_steps'] = 10

224
225
    super(Resnet50KerasBenchmarkReal, self).__init__(
        output_dir=output_dir, default_flags=def_flags)