hooks_test.py 5.48 KB
Newer Older
Yanhui Liang's avatar
Yanhui Liang committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Tests for hooks."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import time

Karmel Allison's avatar
Karmel Allison committed
24
import tensorflow as tf  # pylint: disable=g-bad-import-order
Yanhui Liang's avatar
Yanhui Liang committed
25

26
from official.utils.logs import hooks
Karmel Allison's avatar
Karmel Allison committed
27
from official.utils.testing import mock_lib
Yanhui Liang's avatar
Yanhui Liang committed
28

29
tf.compat.v1.logging.set_verbosity(tf.compat.v1.logging.DEBUG)
Yanhui Liang's avatar
Yanhui Liang committed
30
31
32


class ExamplesPerSecondHookTest(tf.test.TestCase):
33
34
  """Tests for the ExamplesPerSecondHook.

35
36
37
38
  In the test, we explicitly run global_step tensor after train_op in order to
  keep the global_step value and the train_op (which increase the glboal_step
  by 1) consistent. This is to correct the discrepancies in reported global_step
  value when running on GPUs.
39
  """
Yanhui Liang's avatar
Yanhui Liang committed
40
41
42

  def setUp(self):
    """Mock out logging calls to verify if correct info is being monitored."""
Karmel Allison's avatar
Karmel Allison committed
43
    self._logger = mock_lib.MockBenchmarkLogger()
Yanhui Liang's avatar
Yanhui Liang committed
44
45
46

    self.graph = tf.Graph()
    with self.graph.as_default():
47
48
49
50
      tf.compat.v1.train.create_global_step()
      self.train_op = tf.compat.v1.assign_add(
          tf.compat.v1.train.get_global_step(), 1)
      self.global_step = tf.compat.v1.train.get_global_step()
Yanhui Liang's avatar
Yanhui Liang committed
51
52
53
54
55
56

  def test_raise_in_both_secs_and_steps(self):
    with self.assertRaises(ValueError):
      hooks.ExamplesPerSecondHook(
          batch_size=256,
          every_n_steps=10,
Karmel Allison's avatar
Karmel Allison committed
57
58
          every_n_secs=20,
          metric_logger=self._logger)
Yanhui Liang's avatar
Yanhui Liang committed
59
60
61
62
63
64

  def test_raise_in_none_secs_and_steps(self):
    with self.assertRaises(ValueError):
      hooks.ExamplesPerSecondHook(
          batch_size=256,
          every_n_steps=None,
Karmel Allison's avatar
Karmel Allison committed
65
66
          every_n_secs=None,
          metric_logger=self._logger)
Yanhui Liang's avatar
Yanhui Liang committed
67

68
  def _validate_log_every_n_steps(self, every_n_steps, warm_steps):
Yanhui Liang's avatar
Yanhui Liang committed
69
70
71
    hook = hooks.ExamplesPerSecondHook(
        batch_size=256,
        every_n_steps=every_n_steps,
Karmel Allison's avatar
Karmel Allison committed
72
73
        warm_steps=warm_steps,
        metric_logger=self._logger)
Yanhui Liang's avatar
Yanhui Liang committed
74

75
76
    with tf.compat.v1.train.MonitoredSession(
        tf.compat.v1.train.ChiefSessionCreator(), [hook]) as mon_sess:
77
78
79
80
81
82
83
84
      for _ in range(every_n_steps):
        # Explicitly run global_step after train_op to get the accurate
        # global_step value
        mon_sess.run(self.train_op)
        mon_sess.run(self.global_step)
        # Nothing should be in the list yet
        self.assertFalse(self._logger.logged_metric)

Yanhui Liang's avatar
Yanhui Liang committed
85
      mon_sess.run(self.train_op)
86
      global_step_val = mon_sess.run(self.global_step)
Yanhui Liang's avatar
Yanhui Liang committed
87

88
89
90
91
92
      if global_step_val > warm_steps:
        self._assert_metrics()
      else:
        # Nothing should be in the list yet
        self.assertFalse(self._logger.logged_metric)
Karmel Allison's avatar
Karmel Allison committed
93

94
95
96
97
      # Add additional run to verify proper reset when called multiple times.
      prev_log_len = len(self._logger.logged_metric)
      mon_sess.run(self.train_op)
      global_step_val = mon_sess.run(self.global_step)
Yanhui Liang's avatar
Yanhui Liang committed
98

99
100
101
102
103
104
      if every_n_steps == 1 and global_step_val > warm_steps:
        # Each time, we log two additional metrics. Did exactly 2 get added?
        self.assertEqual(len(self._logger.logged_metric), prev_log_len + 2)
      else:
        # No change in the size of the metric list.
        self.assertEqual(len(self._logger.logged_metric), prev_log_len)
Yanhui Liang's avatar
Yanhui Liang committed
105
106

  def test_examples_per_sec_every_1_steps(self):
107
108
    with self.graph.as_default():
      self._validate_log_every_n_steps(1, 0)
Yanhui Liang's avatar
Yanhui Liang committed
109
110

  def test_examples_per_sec_every_5_steps(self):
111
112
    with self.graph.as_default():
      self._validate_log_every_n_steps(5, 0)
Yanhui Liang's avatar
Yanhui Liang committed
113
114

  def test_examples_per_sec_every_1_steps_with_warm_steps(self):
115
116
    with self.graph.as_default():
      self._validate_log_every_n_steps(1, 10)
Yanhui Liang's avatar
Yanhui Liang committed
117
118

  def test_examples_per_sec_every_5_steps_with_warm_steps(self):
119
120
    with self.graph.as_default():
      self._validate_log_every_n_steps(5, 10)
Yanhui Liang's avatar
Yanhui Liang committed
121

122
  def _validate_log_every_n_secs(self, every_n_secs):
Yanhui Liang's avatar
Yanhui Liang committed
123
124
125
    hook = hooks.ExamplesPerSecondHook(
        batch_size=256,
        every_n_steps=None,
Karmel Allison's avatar
Karmel Allison committed
126
127
        every_n_secs=every_n_secs,
        metric_logger=self._logger)
Yanhui Liang's avatar
Yanhui Liang committed
128

129
130
    with tf.compat.v1.train.MonitoredSession(
        tf.compat.v1.train.ChiefSessionCreator(), [hook]) as mon_sess:
131
132
133
134
135
136
137
      # Explicitly run global_step after train_op to get the accurate
      # global_step value
      mon_sess.run(self.train_op)
      mon_sess.run(self.global_step)
      # Nothing should be in the list yet
      self.assertFalse(self._logger.logged_metric)
      time.sleep(every_n_secs)
Yanhui Liang's avatar
Yanhui Liang committed
138

139
140
141
      mon_sess.run(self.train_op)
      mon_sess.run(self.global_step)
      self._assert_metrics()
Yanhui Liang's avatar
Yanhui Liang committed
142
143

  def test_examples_per_sec_every_1_secs(self):
144
145
    with self.graph.as_default():
      self._validate_log_every_n_secs(1)
Yanhui Liang's avatar
Yanhui Liang committed
146
147

  def test_examples_per_sec_every_5_secs(self):
148
149
    with self.graph.as_default():
      self._validate_log_every_n_secs(5)
Yanhui Liang's avatar
Yanhui Liang committed
150

Karmel Allison's avatar
Karmel Allison committed
151
152
153
154
155
  def _assert_metrics(self):
    metrics = self._logger.logged_metric
    self.assertEqual(metrics[-2]["name"], "average_examples_per_sec")
    self.assertEqual(metrics[-1]["name"], "current_examples_per_sec")

Yanhui Liang's avatar
Yanhui Liang committed
156

Karmel Allison's avatar
Karmel Allison committed
157
if __name__ == "__main__":
Yanhui Liang's avatar
Yanhui Liang committed
158
  tf.test.main()