ncf_keras_benchmark.py 14.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Executes Keras benchmarks and accuracy tests."""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import time

from absl import flags
24
from absl import logging
25
from absl.testing import flagsaver
Hongkun Yu's avatar
Hongkun Yu committed
26
import tensorflow as tf
27
from official.benchmark import benchmark_wrappers
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
28
from official.benchmark import owner_utils
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
29
from official.benchmark.perfzero_benchmark import PerfZeroBenchmark
30
31
32
33
34
from official.recommendation import ncf_common
from official.recommendation import ncf_keras_main
from official.utils.flags import core

FLAGS = flags.FLAGS
Toby Boyd's avatar
Toby Boyd committed
35
NCF_DATA_DIR_NAME = 'movielens_data'
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
36
NCF_TF_REGRESSION_DATA_DIR_NAME = 'gs://tf-regression/ncf/data'
Toby Boyd's avatar
Toby Boyd committed
37

38

A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
39
class NCFKerasBenchmarkBase(PerfZeroBenchmark):
40
41
  """Base class for NCF model benchmark."""

A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
42
43
44
45
  def __init__(self, output_dir=None, default_flags=None, **kwargs):
    super(NCFKerasBenchmarkBase, self).__init__(output_dir, default_flags,
                                                **kwargs)

A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
46
47
    # Run all benchmarks with ml_perf flag.
    self.default_flags['ml_perf'] = True
48
49
50

  def _setup(self):
    """Sets up and resets flags before each test."""
51
    logging.set_verbosity(logging.INFO)
52
    if NCFKerasBenchmarkBase.local_flags is None:
Toby Boyd's avatar
Toby Boyd committed
53
      ncf_common.define_ncf_flags()
54
55
56
57
      # Loads flags to get defaults to then override. List cannot be empty.
      flags.FLAGS(['foo'])
      core.set_defaults(**self.default_flags)
      saved_flag_values = flagsaver.save_flag_values()
58
      NCFKerasBenchmarkBase.local_flags = saved_flag_values
59
    else:
60
      flagsaver.restore_flag_values(NCFKerasBenchmarkBase.local_flags)
61

62
  @benchmark_wrappers.enable_runtime_flags
Toby Boyd's avatar
Toby Boyd committed
63
  def _run_and_report_benchmark(self, hr_at_10_min=0, hr_at_10_max=0):
64
65
66
67
    start_time_sec = time.time()
    stats = ncf_keras_main.run_ncf(FLAGS)
    wall_time_sec = time.time() - start_time_sec

Toby Boyd's avatar
Toby Boyd committed
68
    metrics = []
Hongkun Yu's avatar
Hongkun Yu committed
69
70
71
72
    metrics.append({
        'name': 'exp_per_second',
        'value': stats['avg_exp_per_second']
    })
73

Toby Boyd's avatar
Toby Boyd committed
74
    if hr_at_10_min > 0:
Hongkun Yu's avatar
Hongkun Yu committed
75
76
77
78
79
80
      metrics.append({
          'name': 'hr_at_10',
          'value': stats['eval_hit_rate'],
          'min_value': hr_at_10_min,
          'max_value': hr_at_10_max
      })
Toby Boyd's avatar
Toby Boyd committed
81

Hongkun Yu's avatar
Hongkun Yu committed
82
      metrics.append({'name': 'train_loss', 'value': stats['loss']})
Toby Boyd's avatar
Toby Boyd committed
83
84

    self.report_benchmark(iters=-1, wall_time=wall_time_sec, metrics=metrics)
85
86


87
class NCFKerasAccuracy(NCFKerasBenchmarkBase):
88
89
90
91
  """Benchmark NCF model using real data."""

  def __init__(self,
               output_dir=None,
Toby Boyd's avatar
Toby Boyd committed
92
               root_data_dir=None,
93
94
               default_flags=None,
               **kwargs):
Hongkun Yu's avatar
Hongkun Yu committed
95
    root_data_dir = root_data_dir if root_data_dir else ''
96
97
98
    default_flags = {}
    default_flags['dataset'] = 'ml-20m'
    default_flags['num_gpus'] = 1
99
    default_flags['train_epochs'] = 10
100
    default_flags['clean'] = True
101
    default_flags['batch_size'] = 99000
102
103
104
105
106
107
108
    default_flags['learning_rate'] = 0.00382059
    default_flags['beta1'] = 0.783529
    default_flags['beta2'] = 0.909003
    default_flags['epsilon'] = 1.45439e-07
    default_flags['layers'] = [256, 256, 128, 64]
    default_flags['num_factors'] = 64
    default_flags['hr_threshold'] = 0.635
109
    default_flags['ml_perf'] = True
110
    default_flags['use_synthetic_data'] = False
Toby Boyd's avatar
Toby Boyd committed
111
    default_flags['data_dir'] = os.path.join(root_data_dir, NCF_DATA_DIR_NAME)
112

113
    super(NCFKerasAccuracy, self).__init__(
Hongkun Yu's avatar
Hongkun Yu committed
114
        output_dir=output_dir, default_flags=default_flags, **kwargs)
115

Toby Boyd's avatar
Toby Boyd committed
116
117
  def _run_and_report_benchmark_mlperf_like(self):
    """Run test and report results.
Toby Boyd's avatar
Toby Boyd committed
118

Toby Boyd's avatar
Toby Boyd committed
119
120
121
    Note: MLPerf like tests are not tuned to hit a specific hr@10 value, but
    we want it recorded.
    """
122
    self._run_and_report_benchmark(hr_at_10_min=0.61)
Toby Boyd's avatar
Toby Boyd committed
123

124
  def _run_and_report_benchmark(self, hr_at_10_min=0.630, hr_at_10_max=0.645):
Toby Boyd's avatar
Toby Boyd committed
125
    """Run test and report results.
Toby Boyd's avatar
Toby Boyd committed
126

Toby Boyd's avatar
Toby Boyd committed
127
128
129
130
131
132
133
134
    Note: Target is 0.635, but some runs are below that level. Until we have
    multi-run tests, we have to accept a lower target.

    Args:
      hr_at_10_min: Minimum acceptable hr@10 value.
      hr_at_10_max: Maximum acceptable hr@10 value.
    """
    super(NCFKerasAccuracy, self)._run_and_report_benchmark(
Hongkun Yu's avatar
Hongkun Yu committed
135
        hr_at_10_min=hr_at_10_min, hr_at_10_max=hr_at_10_max)
136

A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
  def _set_8_gpu_defaults(self):
    FLAGS.num_gpus = 8
    FLAGS.learning_rate = 0.0045
    FLAGS.beta1 = 0.25
    FLAGS.beta2 = 0.5
    FLAGS.epsilon = 1e-8
    FLAGS.train_epochs = 14
    FLAGS.batch_size = 99000
    FLAGS.eval_batch_size = 160000
    FLAGS.train_dataset_path = os.path.join(NCF_TF_REGRESSION_DATA_DIR_NAME,
                                            'training_cycle_*/*')
    FLAGS.eval_dataset_path = os.path.join(NCF_TF_REGRESSION_DATA_DIR_NAME,
                                           'eval_data/*')
    FLAGS.input_meta_data_path = os.path.join(NCF_TF_REGRESSION_DATA_DIR_NAME,
                                              'metadata')
    FLAGS.data_dir = NCF_TF_REGRESSION_DATA_DIR_NAME

154
  def benchmark_1_gpu_early_stop(self):
155
    self._setup()
156
    FLAGS.early_stopping = True
157
158
    self._run_and_report_benchmark()

159
160
161
162
163
164
  def benchmark_1_gpu_no_dist_strat_early_stop(self):
    self._setup()
    FLAGS.distribution_strategy = 'off'
    FLAGS.early_stopping = True
    self._run_and_report_benchmark()

165
166
167
168
169
170
171
172
173
174
175
176
177
  def benchmark_1_gpu_no_dist_strat_run_eagerly_early_stop(self):
    self._setup()
    FLAGS.distribution_strategy = 'off'
    FLAGS.early_stopping = True
    FLAGS.run_eagerly = True
    self._run_and_report_benchmark()

  def benchmark_xla_1_gpu_early_stop(self):
    self._setup()
    FLAGS.early_stopping = True
    FLAGS.enable_xla = True
    self._run_and_report_benchmark()

178
179
180
181
182
183
  def benchmark_1_gpu_ctl_early_stop(self):
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.early_stopping = True
    self._run_and_report_benchmark()

184
185
186
187
188
189
190
  def benchmark_1_gpu_ctl_run_eagerly_early_stop(self):
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.early_stopping = True
    FLAGS.run_eagerly = True
    self._run_and_report_benchmark()

191
192
193
194
195
196
197
  def benchmark_xla_1_gpu_ctl_early_stop(self):
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.early_stopping = True
    FLAGS.enable_xla = True
    self._run_and_report_benchmark()

198
199
200
201
  def benchmark_2_gpus_early_stop(self):
    self._setup()
    FLAGS.early_stopping = True
    FLAGS.num_gpus = 2
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
202
    FLAGS.eval_batch_size = 160000
203
    self._run_and_report_benchmark()
204

205
  def benchmark_2_gpus_ctl_early_stop(self):
206
    """NCF with custom training loop. Works only in TF 2.0."""
207
208
209
210
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.early_stopping = True
    FLAGS.num_gpus = 2
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
211
    FLAGS.eval_batch_size = 160000
212
213
    self._run_and_report_benchmark()

Hongkun Yu's avatar
Hongkun Yu committed
214

215
#############################################
216
# Tests below with mlperf in the test name are of two types:
217
218
219
220
221
222
223
#  1) 1 GPU tests are based on MLPerf 0.5 and the TensorFlow pulled submission.
#  2) 8 GPU tests are based on MLPerf 0.5 and use NVIDIA's hyper parameters.
#
# The purpose of both is to get a number to compare to existing results. To do
# this the number of epochs is held constant rather than a race to a given
# accuracy. The accuracy validation is done by the "early_stop" tests.
#############################################
224
225

  def benchmark_1_gpu_mlperf_like(self):
226
    """1 GPU using keras fit/compile."""
227
228
    self._setup()
    FLAGS.train_epochs = 7
Toby Boyd's avatar
Toby Boyd committed
229
    self._run_and_report_benchmark_mlperf_like()
230
231

  def benchmark_1_gpu_no_dist_strat_mlperf_like(self):
232
    """1 GPU using compile/fit without dist_strat."""
233
234
235
    self._setup()
    FLAGS.train_epochs = 7
    FLAGS.distribution_strategy = 'off'
Toby Boyd's avatar
Toby Boyd committed
236
    self._run_and_report_benchmark_mlperf_like()
237
238
239
240
241
242

  def benchmark_1_gpu_no_dist_strat_run_eagerly_mlperf_like(self):
    self._setup()
    FLAGS.train_epochs = 7
    FLAGS.distribution_strategy = 'off'
    FLAGS.run_eagerly = True
Toby Boyd's avatar
Toby Boyd committed
243
    self._run_and_report_benchmark_mlperf_like()
244
245

  def benchmark_xla_1_gpu_mlperf_like(self):
246
    """1 GPU using compile/fit with XLA."""
247
248
    self._setup()
    FLAGS.train_epochs = 7
249
    FLAGS.enable_xla = True
Toby Boyd's avatar
Toby Boyd committed
250
    self._run_and_report_benchmark_mlperf_like()
251

252
253
254
255
256
  def benchmark_1_gpu_ctl_mlperf_like(self):
    """1 GPU using CTL."""
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.train_epochs = 7
Toby Boyd's avatar
Toby Boyd committed
257
    self._run_and_report_benchmark_mlperf_like()
258

Nimit Nigania's avatar
Nimit Nigania committed
259
  def benchmark_1_gpu_ctl_fp16_mlperf_like(self):
Tomasz Grel's avatar
Tomasz Grel committed
260
    """1 GPU using CTL and FP16."""
Nimit Nigania's avatar
Nimit Nigania committed
261
262
263
264
265
266
267
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.train_epochs = 7
    FLAGS.dtype = 'fp16'
    FLAGS.loss_scale = 8192
    self._run_and_report_benchmark_mlperf_like()

Tomasz Grel's avatar
Tomasz Grel committed
268
269
270
271
272
273
274
275
  def benchmark_1_gpu_fp16_mlperf_like(self):
    """1 GPU using FP16."""
    self._setup()
    FLAGS.train_epochs = 7
    FLAGS.dtype = 'fp16'
    FLAGS.loss_scale = 8192
    self._run_and_report_benchmark_mlperf_like()

276
277
278
279
280
281
282
283
  def benchmark_1_gpu_ctl_run_eagerly_mlperf_like(self):
    """1 GPU using CTL with eager and distribution strategy."""
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.run_eagerly = True
    FLAGS.train_epochs = 7
    self._run_and_report_benchmark()

284
285
  def benchmark_xla_1_gpu_ctl_mlperf_like(self):
    """1 GPU using CTL with XLA."""
286
287
    self._setup()
    FLAGS.keras_use_ctl = True
288
289
    FLAGS.enable_xla = True
    FLAGS.train_epochs = 7
Toby Boyd's avatar
Toby Boyd committed
290
    self._run_and_report_benchmark_mlperf_like()
291

Tomasz Grel's avatar
Tomasz Grel committed
292
293
294
295
296
297
298
299
300
  def benchmark_xla_1_gpu_fp16_mlperf_like(self):
    """1 GPU using with XLA and FP16."""
    self._setup()
    FLAGS.enable_xla = True
    FLAGS.train_epochs = 7
    FLAGS.dtype = 'fp16'
    FLAGS.loss_scale = 8192
    self._run_and_report_benchmark_mlperf_like()

Nimit Nigania's avatar
Nimit Nigania committed
301
  def benchmark_xla_1_gpu_ctl_fp16_mlperf_like(self):
Tomasz Grel's avatar
Tomasz Grel committed
302
    """1 GPU using CTL with XLA and FP16."""
Nimit Nigania's avatar
Nimit Nigania committed
303
304
305
306
307
308
309
310
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.enable_xla = True
    FLAGS.train_epochs = 7
    FLAGS.dtype = 'fp16'
    FLAGS.loss_scale = 8192
    self._run_and_report_benchmark_mlperf_like()

311
312
313
  def benchmark_8_gpu_mlperf_like(self):
    """8 GPU using keras fit/compile."""
    self._setup()
314
315
316
    FLAGS.num_gpus = 8
    FLAGS.train_epochs = 17
    FLAGS.batch_size = 1048576
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
317
    FLAGS.eval_batch_size = 160000
318
319
320
321
    FLAGS.learning_rate = 0.0045
    FLAGS.beta1 = 0.25
    FLAGS.beta2 = 0.5
    FLAGS.epsilon = 1e-8
Toby Boyd's avatar
Toby Boyd committed
322
    self._run_and_report_benchmark_mlperf_like()
323

324
325
326
327
328
329
330
  def benchmark_8_gpu_ctl_mlperf_like(self):
    """8 GPU using CTL."""
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.num_gpus = 8
    FLAGS.train_epochs = 17
    FLAGS.batch_size = 1048576
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
331
    FLAGS.eval_batch_size = 160000
332
333
334
335
    FLAGS.learning_rate = 0.0045
    FLAGS.beta1 = 0.25
    FLAGS.beta2 = 0.5
    FLAGS.epsilon = 1e-8
Toby Boyd's avatar
Toby Boyd committed
336
    self._run_and_report_benchmark_mlperf_like()
337

A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
338
339
340
  def benchmark_8_gpu_tf_data_ctl_mlperf_like(self):
    """8 GPU using CTL."""
    self._setup()
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
341
    self._set_8_gpu_defaults()
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
342
343
344
    FLAGS.keras_use_ctl = True
    self._run_and_report_benchmark_mlperf_like()

Tomasz Grel's avatar
Tomasz Grel committed
345
  def benchmark_8_gpu_tf_data_fp16_mlperf_like(self):
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
346
    """8 GPU FP16."""
Tomasz Grel's avatar
Tomasz Grel committed
347
    self._setup()
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
348
    self._set_8_gpu_defaults()
Tomasz Grel's avatar
Tomasz Grel committed
349
350
351
352
    FLAGS.dtype = 'fp16'
    FLAGS.loss_scale = 8192
    self._run_and_report_benchmark_mlperf_like()

A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
353
  def benchmark_8_gpu_tf_data_ctl_fp16_mlperf_like(self):
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
354
    """8 GPU FP16 using CTL."""
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
355
    self._setup()
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
356
    self._set_8_gpu_defaults()
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
357
358
359
360
    FLAGS.keras_use_ctl = True
    FLAGS.dtype = 'fp16'
    FLAGS.loss_scale = 8192
    self._run_and_report_benchmark_mlperf_like()
361

362

A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
class NCFKerasBenchmarkReal(NCFKerasBenchmarkBase):
  """NCF Keras throughput benchmarks."""

  def __init__(self,
               output_dir=None,
               root_data_dir=None,
               default_flags=None,
               **kwargs):

    root_data_dir = root_data_dir if root_data_dir else ''
    default_flags = {}
    default_flags['dataset'] = 'ml-20m'
    default_flags['num_gpus'] = 1
    default_flags['train_epochs'] = 14
    default_flags['clean'] = True
    default_flags['batch_size'] = 99000
    default_flags['eval_batch_size'] = 160000
    default_flags['learning_rate'] = 0.00382059
    default_flags['beta1'] = 0.783529
    default_flags['beta2'] = 0.909003
    default_flags['epsilon'] = 1.45439e-07
    default_flags['layers'] = [256, 256, 128, 64]
    default_flags['num_factors'] = 64
    default_flags['hr_threshold'] = 0.635
    default_flags['ml_perf'] = True
    default_flags['use_synthetic_data'] = False
    default_flags['train_dataset_path'] = os.path.join(
        NCF_TF_REGRESSION_DATA_DIR_NAME, 'training_cycle_*/*')
    default_flags['eval_dataset_path'] = os.path.join(
        NCF_TF_REGRESSION_DATA_DIR_NAME, 'eval_data/*')
    default_flags['input_meta_data_path'] = os.path.join(
        NCF_TF_REGRESSION_DATA_DIR_NAME, 'metadata')
    default_flags['data_dir'] = NCF_TF_REGRESSION_DATA_DIR_NAME

    super(NCFKerasBenchmarkReal, self).__init__(
        output_dir=output_dir, default_flags=default_flags, **kwargs)

  def benchmark_2x2_tpu(self):
    """2x2 TPU using CTL with distribution strategy."""
    self._setup()
    FLAGS.distribution_strategy = 'tpu'
    FLAGS.keras_use_ctl = True
    FLAGS.num_gpus = 0
    FLAGS.train_epochs = 1
    self._run_and_report_benchmark()

A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
409
410
411
412
413
414
415
416
417
418
419
  @owner_utils.Owner('tf-graph-compiler')
  def benchmark_2x2_tpu_mlir(self):
    """2x2 TPU using CTL with distribution strategy using the MLIR bridge."""
    self._setup()
    FLAGS.distribution_strategy = 'tpu'
    FLAGS.keras_use_ctl = True
    FLAGS.num_gpus = 0
    FLAGS.train_epochs = 1
    tf.config.experimental.enable_mlir_bridge()
    self._run_and_report_benchmark()

A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
420

421
class NCFKerasSynth(NCFKerasBenchmarkBase):
422
423
  """Benchmark NCF model using synthetic data."""

Hongkun Yu's avatar
Hongkun Yu committed
424
  def __init__(self, output_dir=None, default_flags=None, **kwargs):
425
426
427
428

    default_flags = {}
    default_flags['dataset'] = 'ml-20m'
    default_flags['num_gpus'] = 1
429
430
    default_flags['train_epochs'] = 8
    default_flags['batch_size'] = 99000
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
431
    default_flags['eval_batch_size'] = 160000
432
433
434
435
436
437
438
439
440
    default_flags['learning_rate'] = 0.00382059
    default_flags['beta1'] = 0.783529
    default_flags['beta2'] = 0.909003
    default_flags['epsilon'] = 1.45439e-07
    default_flags['layers'] = [256, 256, 128, 64]
    default_flags['num_factors'] = 64
    default_flags['hr_threshold'] = 0.635
    default_flags['use_synthetic_data'] = True

441
    super(NCFKerasSynth, self).__init__(
Hongkun Yu's avatar
Hongkun Yu committed
442
        output_dir=output_dir, default_flags=default_flags, **kwargs)
443
444
445
446

  def benchmark_1_gpu(self):
    self._setup()
    self._run_and_report_benchmark()
447
448
449
450
451

  def benchmark_2_gpus(self):
    self._setup()
    FLAGS.num_gpus = 2
    self._run_and_report_benchmark()
David Chen's avatar
David Chen committed
452
453
454
455


if __name__ == '__main__':
  tf.test.main()