flags_test.py 5.29 KB
Newer Older
1
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
Taylor Robie's avatar
Taylor Robie committed
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

import unittest

18
19
from absl import flags
import tensorflow as tf
Taylor Robie's avatar
Taylor Robie committed
20

21
from official.utils.flags import core as flags_core  # pylint: disable=g-bad-import-order
Taylor Robie's avatar
Taylor Robie committed
22
23


24
def define_flags():
Hongkun Yu's avatar
Hongkun Yu committed
25
26
27
28
29
30
31
  flags_core.define_base(
      clean=True,
      num_gpu=False,
      stop_threshold=True,
      hooks=True,
      train_epochs=True,
      epochs_between_evals=True)
32
  flags_core.define_performance(
Hongkun Yu's avatar
Hongkun Yu committed
33
34
35
36
37
38
      num_parallel_calls=True,
      inter_op=True,
      intra_op=True,
      dynamic_loss_scale=True,
      loss_scale=True,
      synthetic_data=True,
39
      dtype=True)
40
41
  flags_core.define_image()
  flags_core.define_benchmark()
Taylor Robie's avatar
Taylor Robie committed
42
43
44
45


class BaseTester(unittest.TestCase):

46
47
48
49
50
  @classmethod
  def setUpClass(cls):
    super(BaseTester, cls).setUpClass()
    define_flags()

Taylor Robie's avatar
Taylor Robie committed
51
  def test_default_setting(self):
Hongkun Yu's avatar
Hongkun Yu committed
52
    """Test to ensure fields exist and defaults can be set."""
Taylor Robie's avatar
Taylor Robie committed
53
54
55
56
57

    defaults = dict(
        data_dir="dfgasf",
        model_dir="dfsdkjgbs",
        train_epochs=534,
58
        epochs_between_evals=15,
Taylor Robie's avatar
Taylor Robie committed
59
        batch_size=256,
60
        hooks=["LoggingTensorHook"],
Taylor Robie's avatar
Taylor Robie committed
61
62
        num_parallel_calls=18,
        inter_op_parallelism_threads=5,
63
        intra_op_parallelism_threads=10,
Hongkun Yu's avatar
Hongkun Yu committed
64
        data_format="channels_first")
Taylor Robie's avatar
Taylor Robie committed
65

66
67
    flags_core.set_defaults(**defaults)
    flags_core.parse_flags()
Taylor Robie's avatar
Taylor Robie committed
68
69

    for key, value in defaults.items():
70
      assert flags.FLAGS.get_flag_value(name=key, default=None) == value
Taylor Robie's avatar
Taylor Robie committed
71

72
73
74
  def test_benchmark_setting(self):
    defaults = dict(
        hooks=["LoggingMetricHook"],
75
76
        benchmark_log_dir="/tmp/12345",
        gcp_project="project_abc",
77
78
    )

79
80
    flags_core.set_defaults(**defaults)
    flags_core.parse_flags()
81
82

    for key, value in defaults.items():
83
      assert flags.FLAGS.get_flag_value(name=key, default=None) == value
84

Taylor Robie's avatar
Taylor Robie committed
85
  def test_booleans(self):
Hongkun Yu's avatar
Hongkun Yu committed
86
    """Test to ensure boolean flags trigger as expected."""
Taylor Robie's avatar
Taylor Robie committed
87

88
    flags_core.parse_flags([__file__, "--use_synthetic_data"])
Taylor Robie's avatar
Taylor Robie committed
89

90
    assert flags.FLAGS.use_synthetic_data
Taylor Robie's avatar
Taylor Robie committed
91

92
  def test_parse_dtype_info(self):
93
94
    flags_core.parse_flags([__file__, "--dtype", "fp16"])
    self.assertEqual(flags_core.get_tf_dtype(flags.FLAGS), tf.float16)
Hongkun Yu's avatar
Hongkun Yu committed
95
96
    self.assertEqual(
        flags_core.get_loss_scale(flags.FLAGS, default_for_fp16=2), 2)
97

Hongkun Yu's avatar
Hongkun Yu committed
98
99
100
    flags_core.parse_flags([__file__, "--dtype", "fp16", "--loss_scale", "5"])
    self.assertEqual(
        flags_core.get_loss_scale(flags.FLAGS, default_for_fp16=2), 5)
101
102
103

    flags_core.parse_flags(
        [__file__, "--dtype", "fp16", "--loss_scale", "dynamic"])
Hongkun Yu's avatar
Hongkun Yu committed
104
105
    self.assertEqual(
        flags_core.get_loss_scale(flags.FLAGS, default_for_fp16=2), "dynamic")
106
107
108

    flags_core.parse_flags([__file__, "--dtype", "fp32"])
    self.assertEqual(flags_core.get_tf_dtype(flags.FLAGS), tf.float32)
Hongkun Yu's avatar
Hongkun Yu committed
109
110
    self.assertEqual(
        flags_core.get_loss_scale(flags.FLAGS, default_for_fp16=2), 1)
111
112

    flags_core.parse_flags([__file__, "--dtype", "fp32", "--loss_scale", "5"])
Hongkun Yu's avatar
Hongkun Yu committed
113
114
    self.assertEqual(
        flags_core.get_loss_scale(flags.FLAGS, default_for_fp16=2), 5)
115

116
    with self.assertRaises(SystemExit):
117
      flags_core.parse_flags([__file__, "--dtype", "int8"])
118

119
    with self.assertRaises(SystemExit):
Hongkun Yu's avatar
Hongkun Yu committed
120
121
      flags_core.parse_flags(
          [__file__, "--dtype", "fp16", "--loss_scale", "abc"])
122

123
124
125
126
127
128
  def test_get_nondefault_flags_as_str(self):
    defaults = dict(
        clean=True,
        data_dir="abc",
        hooks=["LoggingTensorHook"],
        stop_threshold=1.5,
Hongkun Yu's avatar
Hongkun Yu committed
129
        use_synthetic_data=False)
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
    flags_core.set_defaults(**defaults)
    flags_core.parse_flags()

    expected_flags = ""
    self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags)

    flags.FLAGS.clean = False
    expected_flags += "--noclean"
    self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags)

    flags.FLAGS.data_dir = "xyz"
    expected_flags += " --data_dir=xyz"
    self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags)

    flags.FLAGS.hooks = ["aaa", "bbb", "ccc"]
    expected_flags += " --hooks=aaa,bbb,ccc"
    self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags)

    flags.FLAGS.stop_threshold = 3.
    expected_flags += " --stop_threshold=3.0"
    self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags)

    flags.FLAGS.use_synthetic_data = True
    expected_flags += " --use_synthetic_data"
    self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags)

    # Assert that explicit setting a flag to its default value does not cause it
    # to appear in the string
    flags.FLAGS.use_synthetic_data = False
    expected_flags = expected_flags[:-len(" --use_synthetic_data")]
    self.assertEqual(flags_core.get_nondefault_flags_as_str(), expected_flags)
161

Taylor Robie's avatar
Taylor Robie committed
162
163
164

if __name__ == "__main__":
  unittest.main()