ncf_keras_benchmark.py 14.7 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Executes Keras benchmarks and accuracy tests."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
import time

from absl import flags
from absl.testing import flagsaver
Hongkun Yu's avatar
Hongkun Yu committed
26
import tensorflow as tf
27
28
29
30
31
32

from official.recommendation import ncf_common
from official.recommendation import ncf_keras_main
from official.utils.flags import core

FLAGS = flags.FLAGS
Toby Boyd's avatar
Toby Boyd committed
33
NCF_DATA_DIR_NAME = 'movielens_data'
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
34
NCF_TF_DATA_1M_BATCH_DIR_NAME = 'gs://tf-perfzero-data/movielens_data/ncf_8gpu_1M_batch'
Toby Boyd's avatar
Toby Boyd committed
35

36

37
class NCFKerasBenchmarkBase(tf.test.Benchmark):
38
39
40
41
42
43
44
45
46
47
48
49
  """Base class for NCF model benchmark."""
  local_flags = None

  def __init__(self,
               output_dir=None,
               default_flags=None,
               **kwargs):
    self.output_dir = output_dir
    self.default_flags = default_flags or {}

  def _setup(self):
    """Sets up and resets flags before each test."""
50
    assert tf.version.VERSION.startswith('2.')
51
    tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.DEBUG)
52
    if NCFKerasBenchmarkBase.local_flags is None:
Toby Boyd's avatar
Toby Boyd committed
53
      ncf_common.define_ncf_flags()
54
55
56
57
      # Loads flags to get defaults to then override. List cannot be empty.
      flags.FLAGS(['foo'])
      core.set_defaults(**self.default_flags)
      saved_flag_values = flagsaver.save_flag_values()
58
      NCFKerasBenchmarkBase.local_flags = saved_flag_values
59
    else:
60
      flagsaver.restore_flag_values(NCFKerasBenchmarkBase.local_flags)
61

Toby Boyd's avatar
Toby Boyd committed
62
  def _run_and_report_benchmark(self, hr_at_10_min=0, hr_at_10_max=0):
63
64
65
66
    start_time_sec = time.time()
    stats = ncf_keras_main.run_ncf(FLAGS)
    wall_time_sec = time.time() - start_time_sec

Toby Boyd's avatar
Toby Boyd committed
67
68
69
    metrics = []
    metrics.append({'name': 'exp_per_second',
                    'value': stats['avg_exp_per_second']})
70

Toby Boyd's avatar
Toby Boyd committed
71
72
73
74
75
76
77
78
79
80
    if hr_at_10_min > 0:
      metrics.append({'name': 'hr_at_10',
                      'value': stats['eval_hit_rate'],
                      'min_value': hr_at_10_min,
                      'max_value': hr_at_10_max})

      metrics.append({'name': 'train_loss',
                      'value': stats['loss']})

    self.report_benchmark(iters=-1, wall_time=wall_time_sec, metrics=metrics)
81
82


83
class NCFKerasAccuracy(NCFKerasBenchmarkBase):
84
85
86
87
  """Benchmark NCF model using real data."""

  def __init__(self,
               output_dir=None,
Toby Boyd's avatar
Toby Boyd committed
88
               root_data_dir=None,
89
90
               default_flags=None,
               **kwargs):
Hongkun Yu's avatar
Hongkun Yu committed
91
    root_data_dir = root_data_dir if root_data_dir else ''
92
93
94
    default_flags = {}
    default_flags['dataset'] = 'ml-20m'
    default_flags['num_gpus'] = 1
95
    default_flags['train_epochs'] = 10
96
    default_flags['clean'] = True
97
    default_flags['batch_size'] = 99000
98
99
100
101
102
103
104
    default_flags['learning_rate'] = 0.00382059
    default_flags['beta1'] = 0.783529
    default_flags['beta2'] = 0.909003
    default_flags['epsilon'] = 1.45439e-07
    default_flags['layers'] = [256, 256, 128, 64]
    default_flags['num_factors'] = 64
    default_flags['hr_threshold'] = 0.635
105
    default_flags['ml_perf'] = True
106
    default_flags['use_synthetic_data'] = False
Toby Boyd's avatar
Toby Boyd committed
107
    default_flags['data_dir'] = os.path.join(root_data_dir, NCF_DATA_DIR_NAME)
108

109
    super(NCFKerasAccuracy, self).__init__(
110
111
112
113
        output_dir=output_dir,
        default_flags=default_flags,
        **kwargs)

Toby Boyd's avatar
Toby Boyd committed
114
115
  def _run_and_report_benchmark_mlperf_like(self):
    """Run test and report results.
Toby Boyd's avatar
Toby Boyd committed
116

Toby Boyd's avatar
Toby Boyd committed
117
118
119
    Note: MLPerf like tests are not tuned to hit a specific hr@10 value, but
    we want it recorded.
    """
120
    self._run_and_report_benchmark(hr_at_10_min=0.61)
Toby Boyd's avatar
Toby Boyd committed
121

122
  def _run_and_report_benchmark(self, hr_at_10_min=0.630, hr_at_10_max=0.645):
Toby Boyd's avatar
Toby Boyd committed
123
    """Run test and report results.
Toby Boyd's avatar
Toby Boyd committed
124

Toby Boyd's avatar
Toby Boyd committed
125
126
127
128
129
130
131
132
    Note: Target is 0.635, but some runs are below that level. Until we have
    multi-run tests, we have to accept a lower target.

    Args:
      hr_at_10_min: Minimum acceptable hr@10 value.
      hr_at_10_max: Maximum acceptable hr@10 value.
    """
    super(NCFKerasAccuracy, self)._run_and_report_benchmark(
133
134
        hr_at_10_min=hr_at_10_min,
        hr_at_10_max=hr_at_10_max)
135

136
  def benchmark_1_gpu_early_stop(self):
137
    self._setup()
138
    FLAGS.early_stopping = True
139
140
    self._run_and_report_benchmark()

141
  def benchmark_1_gpu_force_v1_path_early_stop(self):
142
143
    self._setup()
    FLAGS.early_stopping = True
144
    FLAGS.force_v2_in_keras_compile = False
145
146
    self._run_and_report_benchmark()

147
148
149
150
151
152
  def benchmark_1_gpu_no_dist_strat_early_stop(self):
    self._setup()
    FLAGS.distribution_strategy = 'off'
    FLAGS.early_stopping = True
    self._run_and_report_benchmark()

153
  def benchmark_1_gpu_no_dist_strat_force_v1_path_early_stop(self):
154
155
156
    self._setup()
    FLAGS.distribution_strategy = 'off'
    FLAGS.early_stopping = True
157
    FLAGS.force_v2_in_keras_compile = False
158
159
    self._run_and_report_benchmark()

160
161
162
163
164
165
166
167
168
169
170
171
172
  def benchmark_1_gpu_no_dist_strat_run_eagerly_early_stop(self):
    self._setup()
    FLAGS.distribution_strategy = 'off'
    FLAGS.early_stopping = True
    FLAGS.run_eagerly = True
    self._run_and_report_benchmark()

  def benchmark_xla_1_gpu_early_stop(self):
    self._setup()
    FLAGS.early_stopping = True
    FLAGS.enable_xla = True
    self._run_and_report_benchmark()

173
  def benchmark_xla_1_gpu_force_v1_path_early_stop(self):
174
175
176
    self._setup()
    FLAGS.early_stopping = True
    FLAGS.enable_xla = True
177
    FLAGS.force_v2_in_keras_compile = False
178
179
    self._run_and_report_benchmark()

180
181
182
183
184
185
  def benchmark_1_gpu_ctl_early_stop(self):
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.early_stopping = True
    self._run_and_report_benchmark()

186
187
188
189
190
191
192
  def benchmark_1_gpu_ctl_run_eagerly_early_stop(self):
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.early_stopping = True
    FLAGS.run_eagerly = True
    self._run_and_report_benchmark()

193
194
195
196
197
198
199
  def benchmark_xla_1_gpu_ctl_early_stop(self):
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.early_stopping = True
    FLAGS.enable_xla = True
    self._run_and_report_benchmark()

200
201
202
203
  def benchmark_2_gpus_early_stop(self):
    self._setup()
    FLAGS.early_stopping = True
    FLAGS.num_gpus = 2
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
204
    FLAGS.eval_batch_size = 160000
205
    self._run_and_report_benchmark()
206

207
  def benchmark_2_gpus_ctl_early_stop(self):
208
    """NCF with custom training loop. Works only in TF 2.0."""
209
210
211
212
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.early_stopping = True
    FLAGS.num_gpus = 2
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
213
    FLAGS.eval_batch_size = 160000
214
215
    self._run_and_report_benchmark()

216
#############################################
217
# Tests below with mlperf in the test name are of two types:
218
219
220
221
222
223
224
#  1) 1 GPU tests are based on MLPerf 0.5 and the TensorFlow pulled submission.
#  2) 8 GPU tests are based on MLPerf 0.5 and use NVIDIA's hyper parameters.
#
# The purpose of both is to get a number to compare to existing results. To do
# this the number of epochs is held constant rather than a race to a given
# accuracy. The accuracy validation is done by the "early_stop" tests.
#############################################
225
226

  def benchmark_1_gpu_mlperf_like(self):
227
    """1 GPU using keras fit/compile."""
228
229
    self._setup()
    FLAGS.train_epochs = 7
Toby Boyd's avatar
Toby Boyd committed
230
    self._run_and_report_benchmark_mlperf_like()
231

232
  def benchmark_1_gpu_no_dist_strat_force_v1_path_mlperf_like(self):
233
234
235
236
    """1 GPU using compile/fit without dist_strat."""
    self._setup()
    FLAGS.train_epochs = 7
    FLAGS.distribution_strategy = 'off'
237
    FLAGS.force_v2_in_keras_compile = False
238
239
    self._run_and_report_benchmark()

240
  def benchmark_1_gpu_no_dist_strat_mlperf_like(self):
241
    """1 GPU using compile/fit without dist_strat."""
242
243
244
    self._setup()
    FLAGS.train_epochs = 7
    FLAGS.distribution_strategy = 'off'
Toby Boyd's avatar
Toby Boyd committed
245
    self._run_and_report_benchmark_mlperf_like()
246
247
248
249
250
251

  def benchmark_1_gpu_no_dist_strat_run_eagerly_mlperf_like(self):
    self._setup()
    FLAGS.train_epochs = 7
    FLAGS.distribution_strategy = 'off'
    FLAGS.run_eagerly = True
Toby Boyd's avatar
Toby Boyd committed
252
    self._run_and_report_benchmark_mlperf_like()
253
254

  def benchmark_xla_1_gpu_mlperf_like(self):
255
    """1 GPU using compile/fit with XLA."""
256
257
    self._setup()
    FLAGS.train_epochs = 7
258
    FLAGS.enable_xla = True
Toby Boyd's avatar
Toby Boyd committed
259
    self._run_and_report_benchmark_mlperf_like()
260

261
262
263
264
265
  def benchmark_1_gpu_ctl_mlperf_like(self):
    """1 GPU using CTL."""
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.train_epochs = 7
Toby Boyd's avatar
Toby Boyd committed
266
    self._run_and_report_benchmark_mlperf_like()
267

Nimit Nigania's avatar
Nimit Nigania committed
268
  def benchmark_1_gpu_ctl_fp16_mlperf_like(self):
Tomasz Grel's avatar
Tomasz Grel committed
269
    """1 GPU using CTL and FP16."""
Nimit Nigania's avatar
Nimit Nigania committed
270
271
272
273
274
275
276
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.train_epochs = 7
    FLAGS.dtype = 'fp16'
    FLAGS.loss_scale = 8192
    self._run_and_report_benchmark_mlperf_like()

Tomasz Grel's avatar
Tomasz Grel committed
277
278
279
280
281
282
283
284
  def benchmark_1_gpu_fp16_mlperf_like(self):
    """1 GPU using FP16."""
    self._setup()
    FLAGS.train_epochs = 7
    FLAGS.dtype = 'fp16'
    FLAGS.loss_scale = 8192
    self._run_and_report_benchmark_mlperf_like()

285
286
287
288
289
290
291
292
  def benchmark_1_gpu_ctl_run_eagerly_mlperf_like(self):
    """1 GPU using CTL with eager and distribution strategy."""
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.run_eagerly = True
    FLAGS.train_epochs = 7
    self._run_and_report_benchmark()

293
294
  def benchmark_xla_1_gpu_ctl_mlperf_like(self):
    """1 GPU using CTL with XLA."""
295
296
    self._setup()
    FLAGS.keras_use_ctl = True
297
298
    FLAGS.enable_xla = True
    FLAGS.train_epochs = 7
Toby Boyd's avatar
Toby Boyd committed
299
    self._run_and_report_benchmark_mlperf_like()
300

Tomasz Grel's avatar
Tomasz Grel committed
301
302
303
304
305
306
307
308
309
  def benchmark_xla_1_gpu_fp16_mlperf_like(self):
    """1 GPU using with XLA and FP16."""
    self._setup()
    FLAGS.enable_xla = True
    FLAGS.train_epochs = 7
    FLAGS.dtype = 'fp16'
    FLAGS.loss_scale = 8192
    self._run_and_report_benchmark_mlperf_like()

Nimit Nigania's avatar
Nimit Nigania committed
310
  def benchmark_xla_1_gpu_ctl_fp16_mlperf_like(self):
Tomasz Grel's avatar
Tomasz Grel committed
311
    """1 GPU using CTL with XLA and FP16."""
Nimit Nigania's avatar
Nimit Nigania committed
312
313
314
315
316
317
318
319
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.enable_xla = True
    FLAGS.train_epochs = 7
    FLAGS.dtype = 'fp16'
    FLAGS.loss_scale = 8192
    self._run_and_report_benchmark_mlperf_like()

320
321
322
  def benchmark_8_gpu_mlperf_like(self):
    """8 GPU using keras fit/compile."""
    self._setup()
323
324
325
    FLAGS.num_gpus = 8
    FLAGS.train_epochs = 17
    FLAGS.batch_size = 1048576
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
326
    FLAGS.eval_batch_size = 160000
327
328
329
330
    FLAGS.learning_rate = 0.0045
    FLAGS.beta1 = 0.25
    FLAGS.beta2 = 0.5
    FLAGS.epsilon = 1e-8
Toby Boyd's avatar
Toby Boyd committed
331
    self._run_and_report_benchmark_mlperf_like()
332

333
334
  def benchmark_8_gpu_force_v1_path_mlperf_like(self):
    """8 GPU using keras fit/compile v1 codepath."""
335
336
337
338
    self._setup()
    FLAGS.num_gpus = 8
    FLAGS.train_epochs = 17
    FLAGS.batch_size = 1048576
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
339
    FLAGS.eval_batch_size = 160000
340
341
342
343
    FLAGS.learning_rate = 0.0045
    FLAGS.beta1 = 0.25
    FLAGS.beta2 = 0.5
    FLAGS.epsilon = 1e-8
344
    FLAGS.force_v2_in_keras_compile = False
345
    self._run_and_report_benchmark_mlperf_like()
346

347
348
349
350
351
352
353
  def benchmark_8_gpu_ctl_mlperf_like(self):
    """8 GPU using CTL."""
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.num_gpus = 8
    FLAGS.train_epochs = 17
    FLAGS.batch_size = 1048576
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
354
    FLAGS.eval_batch_size = 160000
355
356
357
358
    FLAGS.learning_rate = 0.0045
    FLAGS.beta1 = 0.25
    FLAGS.beta2 = 0.5
    FLAGS.epsilon = 1e-8
Toby Boyd's avatar
Toby Boyd committed
359
    self._run_and_report_benchmark_mlperf_like()
360

A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
  def benchmark_8_gpu_tf_data_ctl_mlperf_like(self):
    """8 GPU using CTL."""
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.num_gpus = 8
    FLAGS.train_epochs = 17
    FLAGS.batch_size = 1048576
    FLAGS.eval_batch_size = 1048000
    FLAGS.learning_rate = 0.0045
    FLAGS.beta1 = 0.25
    FLAGS.beta2 = 0.5
    FLAGS.epsilon = 1e-8
    FLAGS.train_dataset_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "training_cycle_*/*")
    FLAGS.eval_dataset_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "eval_data/*")
    FLAGS.input_meta_data_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "meta_data.json")
    self._run_and_report_benchmark_mlperf_like()

Tomasz Grel's avatar
Tomasz Grel committed
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
  def benchmark_8_gpu_tf_data_fp16_mlperf_like(self):
    """8 GPU FP16"""
    self._setup()
    FLAGS.num_gpus = 8
    FLAGS.train_epochs = 17
    FLAGS.batch_size = 1048576
    FLAGS.eval_batch_size = 1048000
    FLAGS.learning_rate = 0.0045
    FLAGS.beta1 = 0.25
    FLAGS.beta2 = 0.5
    FLAGS.epsilon = 1e-8
    FLAGS.dtype = 'fp16'
    FLAGS.loss_scale = 8192
    FLAGS.train_dataset_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "training_cycle_*/*")
    FLAGS.eval_dataset_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "eval_data/*")
    FLAGS.input_meta_data_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "meta_data.json")
    self._run_and_report_benchmark_mlperf_like()

A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
396
  def benchmark_8_gpu_tf_data_ctl_fp16_mlperf_like(self):
Tomasz Grel's avatar
Tomasz Grel committed
397
    """8 GPU FP16 using CTL"""
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
    self._setup()
    FLAGS.keras_use_ctl = True
    FLAGS.num_gpus = 8
    FLAGS.train_epochs = 17
    FLAGS.batch_size = 1048576
    FLAGS.eval_batch_size = 1048000
    FLAGS.learning_rate = 0.0045
    FLAGS.beta1 = 0.25
    FLAGS.beta2 = 0.5
    FLAGS.epsilon = 1e-8
    FLAGS.dtype = 'fp16'
    FLAGS.loss_scale = 8192
    FLAGS.train_dataset_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "training_cycle_*/*")
    FLAGS.eval_dataset_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "eval_data/*")
    FLAGS.input_meta_data_path = os.path.join(NCF_TF_DATA_1M_BATCH_DIR_NAME, "meta_data.json")
    self._run_and_report_benchmark_mlperf_like()
414

415
class NCFKerasSynth(NCFKerasBenchmarkBase):
416
417
418
419
420
421
422
423
424
425
  """Benchmark NCF model using synthetic data."""

  def __init__(self,
               output_dir=None,
               default_flags=None,
               **kwargs):

    default_flags = {}
    default_flags['dataset'] = 'ml-20m'
    default_flags['num_gpus'] = 1
426
427
    default_flags['train_epochs'] = 8
    default_flags['batch_size'] = 99000
A. Unique TensorFlower's avatar
A. Unique TensorFlower committed
428
    default_flags['eval_batch_size'] = 160000
429
430
431
432
433
434
435
436
437
    default_flags['learning_rate'] = 0.00382059
    default_flags['beta1'] = 0.783529
    default_flags['beta2'] = 0.909003
    default_flags['epsilon'] = 1.45439e-07
    default_flags['layers'] = [256, 256, 128, 64]
    default_flags['num_factors'] = 64
    default_flags['hr_threshold'] = 0.635
    default_flags['use_synthetic_data'] = True

438
    super(NCFKerasSynth, self).__init__(
439
440
441
442
443
444
445
        output_dir=output_dir,
        default_flags=default_flags,
        **kwargs)

  def benchmark_1_gpu(self):
    self._setup()
    self._run_and_report_benchmark()
446
447
448
449
450

  def benchmark_2_gpus(self):
    self._setup()
    FLAGS.num_gpus = 2
    self._run_and_report_benchmark()
David Chen's avatar
David Chen committed
451
452
453
454


if __name__ == '__main__':
  tf.test.main()