hooks_test.py 5.15 KB
Newer Older
Yanhui Liang's avatar
Yanhui Liang committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Tests for hooks."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import time

Karmel Allison's avatar
Karmel Allison committed
24
25
import tensorflow as tf  # pylint: disable=g-bad-import-order
from tensorflow.python.training import monitored_session  # pylint: disable=g-bad-import-order
Yanhui Liang's avatar
Yanhui Liang committed
26

27
from official.utils.logs import hooks
Karmel Allison's avatar
Karmel Allison committed
28
from official.utils.testing import mock_lib
Yanhui Liang's avatar
Yanhui Liang committed
29
30


Karmel Allison's avatar
Karmel Allison committed
31
tf.logging.set_verbosity(tf.logging.DEBUG)
Yanhui Liang's avatar
Yanhui Liang committed
32
33
34


class ExamplesPerSecondHookTest(tf.test.TestCase):
Karmel Allison's avatar
Karmel Allison committed
35
  """Tests for the ExamplesPerSecondHook."""
Yanhui Liang's avatar
Yanhui Liang committed
36
37
38

  def setUp(self):
    """Mock out logging calls to verify if correct info is being monitored."""
Karmel Allison's avatar
Karmel Allison committed
39
    self._logger = mock_lib.MockBenchmarkLogger()
Yanhui Liang's avatar
Yanhui Liang committed
40
41
42
43
44
45
46
47
48
49
50

    self.graph = tf.Graph()
    with self.graph.as_default():
      self.global_step = tf.train.get_or_create_global_step()
      self.train_op = tf.assign_add(self.global_step, 1)

  def test_raise_in_both_secs_and_steps(self):
    with self.assertRaises(ValueError):
      hooks.ExamplesPerSecondHook(
          batch_size=256,
          every_n_steps=10,
Karmel Allison's avatar
Karmel Allison committed
51
52
          every_n_secs=20,
          metric_logger=self._logger)
Yanhui Liang's avatar
Yanhui Liang committed
53
54
55
56
57
58

  def test_raise_in_none_secs_and_steps(self):
    with self.assertRaises(ValueError):
      hooks.ExamplesPerSecondHook(
          batch_size=256,
          every_n_steps=None,
Karmel Allison's avatar
Karmel Allison committed
59
60
          every_n_secs=None,
          metric_logger=self._logger)
Yanhui Liang's avatar
Yanhui Liang committed
61
62
63
64
65

  def _validate_log_every_n_steps(self, sess, every_n_steps, warm_steps):
    hook = hooks.ExamplesPerSecondHook(
        batch_size=256,
        every_n_steps=every_n_steps,
Karmel Allison's avatar
Karmel Allison committed
66
67
        warm_steps=warm_steps,
        metric_logger=self._logger)
Yanhui Liang's avatar
Yanhui Liang committed
68
    hook.begin()
Karmel Allison's avatar
Karmel Allison committed
69
    mon_sess = monitored_session._HookedSession(sess, [hook])  # pylint: disable=protected-access
Yanhui Liang's avatar
Yanhui Liang committed
70
71
72
73
    sess.run(tf.global_variables_initializer())

    for _ in range(every_n_steps):
      mon_sess.run(self.train_op)
Karmel Allison's avatar
Karmel Allison committed
74
75
      # Nothing should be in the list yet
      self.assertFalse(self._logger.logged_metric)
Yanhui Liang's avatar
Yanhui Liang committed
76
77
78

    mon_sess.run(self.train_op)
    global_step_val = sess.run(self.global_step)
Karmel Allison's avatar
Karmel Allison committed
79

Yanhui Liang's avatar
Yanhui Liang committed
80
    if global_step_val > warm_steps:
Karmel Allison's avatar
Karmel Allison committed
81
      self._assert_metrics()
Yanhui Liang's avatar
Yanhui Liang committed
82
    else:
Karmel Allison's avatar
Karmel Allison committed
83
84
      # Nothing should be in the list yet
      self.assertFalse(self._logger.logged_metric)
Yanhui Liang's avatar
Yanhui Liang committed
85
86

    # Add additional run to verify proper reset when called multiple times.
Karmel Allison's avatar
Karmel Allison committed
87
    prev_log_len = len(self._logger.logged_metric)
Yanhui Liang's avatar
Yanhui Liang committed
88
89
90
    mon_sess.run(self.train_op)
    global_step_val = sess.run(self.global_step)
    if every_n_steps == 1 and global_step_val > warm_steps:
Karmel Allison's avatar
Karmel Allison committed
91
92
      # Each time, we log two additional metrics. Did exactly 2 get added?
      self.assertEqual(len(self._logger.logged_metric), prev_log_len + 2)
Yanhui Liang's avatar
Yanhui Liang committed
93
    else:
Karmel Allison's avatar
Karmel Allison committed
94
95
      # No change in the size of the metric list.
      self.assertEqual(len(self._logger.logged_metric), prev_log_len)
Yanhui Liang's avatar
Yanhui Liang committed
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118

    hook.end(sess)

  def test_examples_per_sec_every_1_steps(self):
    with self.graph.as_default(), tf.Session() as sess:
      self._validate_log_every_n_steps(sess, 1, 0)

  def test_examples_per_sec_every_5_steps(self):
    with self.graph.as_default(), tf.Session() as sess:
      self._validate_log_every_n_steps(sess, 5, 0)

  def test_examples_per_sec_every_1_steps_with_warm_steps(self):
    with self.graph.as_default(), tf.Session() as sess:
      self._validate_log_every_n_steps(sess, 1, 10)

  def test_examples_per_sec_every_5_steps_with_warm_steps(self):
    with self.graph.as_default(), tf.Session() as sess:
      self._validate_log_every_n_steps(sess, 5, 10)

  def _validate_log_every_n_secs(self, sess, every_n_secs):
    hook = hooks.ExamplesPerSecondHook(
        batch_size=256,
        every_n_steps=None,
Karmel Allison's avatar
Karmel Allison committed
119
120
        every_n_secs=every_n_secs,
        metric_logger=self._logger)
Yanhui Liang's avatar
Yanhui Liang committed
121
    hook.begin()
Karmel Allison's avatar
Karmel Allison committed
122
    mon_sess = monitored_session._HookedSession(sess, [hook])  # pylint: disable=protected-access
Yanhui Liang's avatar
Yanhui Liang committed
123
124
125
    sess.run(tf.global_variables_initializer())

    mon_sess.run(self.train_op)
Karmel Allison's avatar
Karmel Allison committed
126
127
    # Nothing should be in the list yet
    self.assertFalse(self._logger.logged_metric)
Yanhui Liang's avatar
Yanhui Liang committed
128
129
130
    time.sleep(every_n_secs)

    mon_sess.run(self.train_op)
Karmel Allison's avatar
Karmel Allison committed
131
    self._assert_metrics()
Yanhui Liang's avatar
Yanhui Liang committed
132
133
134
135
136
137
138
139
140
141
142

    hook.end(sess)

  def test_examples_per_sec_every_1_secs(self):
    with self.graph.as_default(), tf.Session() as sess:
      self._validate_log_every_n_secs(sess, 1)

  def test_examples_per_sec_every_5_secs(self):
    with self.graph.as_default(), tf.Session() as sess:
      self._validate_log_every_n_secs(sess, 5)

Karmel Allison's avatar
Karmel Allison committed
143
144
145
146
147
  def _assert_metrics(self):
    metrics = self._logger.logged_metric
    self.assertEqual(metrics[-2]["name"], "average_examples_per_sec")
    self.assertEqual(metrics[-1]["name"], "current_examples_per_sec")

Yanhui Liang's avatar
Yanhui Liang committed
148

Karmel Allison's avatar
Karmel Allison committed
149
if __name__ == "__main__":
Yanhui Liang's avatar
Yanhui Liang committed
150
  tf.test.main()