Internal change

PiperOrigin-RevId: 409182513

official/benchmark/unet3d_benchmark.py

-# Lint as: python3
-# Copyright 2020 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Executes benchmark testing for 3D Unet model."""
-# pylint: disable=line-too-long
-from __future__ import print_function
-
-import functools
-import os
-import time
-from typing import Optional
-
-from absl import flags
-import tensorflow as tf  # pylint: disable=g-bad-import-order
-
-from official.benchmark import benchmark_wrappers
-from official.benchmark import keras_benchmark
-from official.benchmark import owner_utils
-from official.vision.segmentation import unet_main as unet_training_lib
-from official.vision.segmentation import unet_model as unet_model_lib
-
-UNET3D_MIN_ACCURACY = 0.94
-UNET3D_MAX_ACCURACY = 0.98
-UNET_TRAINING_FILES = 'gs://mlcompass-data/unet3d/train_data/*'
-UNET_EVAL_FILES = 'gs://mlcompass-data/unet3d/eval_data/*'
-UNET_MODEL_CONFIG_FILE = 'gs://mlcompass-data/unet3d/config/unet_config.yaml'
-
-FLAGS = flags.FLAGS
-
-
-class Unet3DAccuracyBenchmark(keras_benchmark.KerasBenchmark):
-  """Benchmark accuracy tests for UNet3D model in Keras."""
-
-  def __init__(self,
-               output_dir: Optional[str] = None,
-               root_data_dir: Optional[str] = None,
-               **kwargs):
-    """A benchmark class.
-
-    Args:
-      output_dir: directory where to output e.g. log files
-      root_data_dir: directory under which to look for dataset
-      **kwargs: arbitrary named arguments. This is needed to make the
-        constructor forward compatible in case PerfZero provides more named
-        arguments before updating the constructor.
-    """
-
-    flag_methods = [unet_training_lib.define_unet3d_flags]
-
-    # UNet3D model in Keras."""
-    self.training_file_pattern = UNET_TRAINING_FILES
-    self.eval_file_pattern = UNET_EVAL_FILES
-
-    # TODO(hongjunchoi): Create and use shared config file instead.
-    self.config_file = UNET_MODEL_CONFIG_FILE
-    super(Unet3DAccuracyBenchmark, self).__init__(
-        output_dir=output_dir, flag_methods=flag_methods)
-
-  def _set_benchmark_parameters(self, experiment_name):
-    """Overrides training parameters for benchmark tests."""
-    FLAGS.model_dir = self._get_model_dir(experiment_name)
-    FLAGS.mode = 'train'
-    FLAGS.training_file_pattern = self.training_file_pattern
-    FLAGS.eval_file_pattern = self.eval_file_pattern
-    FLAGS.config_file = self.config_file
-    FLAGS.lr_init_value = 0.00005
-    FLAGS.lr_decay_rate = 0.5
-    FLAGS.epochs = 3
-
-  @benchmark_wrappers.enable_runtime_flags
-  def _run_and_report_benchmark(self,
-                                experiment_name: str,
-                                min_accuracy: float = UNET3D_MIN_ACCURACY,
-                                max_accuracy: float = UNET3D_MAX_ACCURACY,
-                                distribution_strategy: str = 'tpu',
-                                epochs: int = 10,
-                                steps: int = 0,
-                                epochs_between_evals: int = 1,
-                                dtype: str = 'float32',
-                                enable_xla: bool = False,
-                                run_eagerly: bool = False):
-    """Runs and reports the benchmark given the provided configuration."""
-    params = unet_training_lib.extract_params(FLAGS)
-    strategy = unet_training_lib.create_distribution_strategy(params)
-
-    input_dtype = params.dtype
-    if input_dtype == 'float16' or input_dtype == 'bfloat16':
-      tf.keras.mixed_precision.set_global_policy(
-          'mixed_bfloat16' if input_dtype == 'bfloat16' else 'mixed_float16')
-
-    stats = {}
-    start_time_sec = time.time()
-    with strategy.scope():
-      unet_model = unet_model_lib.build_unet_model(params)
-      history = unet_training_lib.train(
-          params, strategy, unet_model,
-          functools.partial(unet_training_lib.get_train_dataset, params),
-          functools.partial(unet_training_lib.get_eval_dataset, params))
-
-      stats['accuracy_top_1'] = history.history['val_metric_accuracy'][-1]
-      stats['training_accuracy_top_1'] = history.history['metric_accuracy'][-1]
-    wall_time_sec = time.time() - start_time_sec
-
-    super(Unet3DAccuracyBenchmark, self)._report_benchmark(
-        stats,
-        wall_time_sec,
-        top_1_min=min_accuracy,
-        top_1_max=max_accuracy,
-        total_batch_size=params.train_batch_size)
-
-  def _get_model_dir(self, folder_name):
-    return os.path.join(self.output_dir, folder_name)
-
-  @owner_utils.Owner('tf-model-garden')
-  def benchmark_4x4_tpu_bf16(self):
-    """Test Keras model with 4x4 TPU, fp16."""
-    experiment_name = 'benchmark_4x4_tpu_fp16'
-    self._setup()
-    self._set_benchmark_parameters(experiment_name)
-    self._run_and_report_benchmark(
-        experiment_name=experiment_name,
-        dtype='bfloat16',
-        distribution_strategy='tpu')
-
-  @owner_utils.Owner('tf-graph-compiler')
-  def benchmark_4x4_tpu_bf16_mlir(self):
-    """Test Keras model with 4x4 TPU, fp16 and MLIR enabled."""
-    experiment_name = 'benchmark_4x4_tpu_fp16_mlir'
-    tf.config.experimental.enable_mlir_bridge()
-    self._setup()
-    self._set_benchmark_parameters(experiment_name)
-    self._run_and_report_benchmark(
-        experiment_name=experiment_name,
-        dtype='bfloat16',
-        distribution_strategy='tpu')
-
-
-if __name__ == '__main__':
-  tf.test.main()