census_test.py 5.54 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import os
21

Karmel Allison's avatar
Karmel Allison committed
22
import tensorflow as tf  # pylint: disable=g-bad-import-order
23

24
from official.utils.testing import integration
25
26
27
from official.wide_deep import census_dataset
from official.wide_deep import census_main
from official.wide_deep import wide_deep_run_loop
28
29
30
31

tf.logging.set_verbosity(tf.logging.ERROR)

TEST_INPUT = ('18,Self-emp-not-inc,987,Bachelors,12,Married-civ-spouse,abc,'
Karmel Allison's avatar
Karmel Allison committed
32
              'Husband,zyx,wvu,34,56,78,tsr,<=50K')
33
34
35
36
37
38
39
40
41
42
43
44
45
46

TEST_INPUT_VALUES = {
    'age': 18,
    'education_num': 12,
    'capital_gain': 34,
    'capital_loss': 56,
    'hours_per_week': 78,
    'education': 'Bachelors',
    'marital_status': 'Married-civ-spouse',
    'relationship': 'Husband',
    'workclass': 'Self-emp-not-inc',
    'occupation': 'abc',
}

47
TEST_CSV = os.path.join(os.path.dirname(__file__), 'census_test.csv')
48
49
50


class BaseTest(tf.test.TestCase):
Karmel Allison's avatar
Karmel Allison committed
51
  """Tests for Wide Deep model."""
52

53
54
55
  @classmethod
  def setUpClass(cls):  # pylint: disable=invalid-name
    super(BaseTest, cls).setUpClass()
56
    census_main.define_census_flags()
57

58
59
60
61
62
63
64
  def setUp(self):
    # Create temporary CSV file
    self.temp_dir = self.get_temp_dir()
    self.input_csv = os.path.join(self.temp_dir, 'test.csv')
    with tf.gfile.Open(self.input_csv, 'w') as temp_csv:
      temp_csv.write(TEST_INPUT)

65
66
67
68
    with tf.gfile.Open(TEST_CSV, "r") as temp_csv:
      test_csv_contents = temp_csv.read()

    # Used for end-to-end tests.
69
    for fname in [census_dataset.TRAINING_FILE, census_dataset.EVAL_FILE]:
70
71
72
      with tf.gfile.Open(os.path.join(self.temp_dir, fname), 'w') as test_csv:
        test_csv.write(test_csv_contents)

73
  def test_input_fn(self):
74
    dataset = census_dataset.input_fn(self.input_csv, 1, False, 1)
Neal Wu's avatar
Neal Wu committed
75
76
    features, labels = dataset.make_one_shot_iterator().get_next()

77
    with self.test_session() as sess:
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
      features, labels = sess.run((features, labels))

      # Compare the two features dictionaries.
      for key in TEST_INPUT_VALUES:
        self.assertTrue(key in features)
        self.assertEqual(len(features[key]), 1)
        feature_value = features[key][0]

        # Convert from bytes to string for Python 3.
        if isinstance(feature_value, bytes):
          feature_value = feature_value.decode()

        self.assertEqual(TEST_INPUT_VALUES[key], feature_value)

      self.assertFalse(labels)

  def build_and_test_estimator(self, model_type):
    """Ensure that model trains and minimizes loss."""
96
97
    model = census_main.build_estimator(
        self.temp_dir, model_type,
98
99
        model_column_fn=census_dataset.build_model_columns,
        inter_op=0, intra_op=0)
100
101

    # Train for 1 step to initialize model and evaluate initial loss
Karmel Allison's avatar
Karmel Allison committed
102
103
    def get_input_fn(num_epochs, shuffle, batch_size):
      def input_fn():
104
        return census_dataset.input_fn(
Karmel Allison's avatar
Karmel Allison committed
105
106
107
108
109
110
            TEST_CSV, num_epochs=num_epochs, shuffle=shuffle,
            batch_size=batch_size)
      return input_fn

    model.train(input_fn=get_input_fn(1, True, 1), steps=1)
    initial_results = model.evaluate(input_fn=get_input_fn(1, False, 1))
111

Neal Wu's avatar
Neal Wu committed
112
    # Train for 100 epochs at batch size 3 and evaluate final loss
Karmel Allison's avatar
Karmel Allison committed
113
114
    model.train(input_fn=get_input_fn(100, True, 3))
    final_results = model.evaluate(input_fn=get_input_fn(1, False, 1))
115
116
117

    print('%s initial results:' % model_type, initial_results)
    print('%s final results:' % model_type, final_results)
Neal Wu's avatar
Neal Wu committed
118
119

    # Ensure loss has decreased, while accuracy and both AUCs have increased.
120
    self.assertLess(final_results['loss'], initial_results['loss'])
Neal Wu's avatar
Neal Wu committed
121
122
123
124
    self.assertGreater(final_results['auc'], initial_results['auc'])
    self.assertGreater(final_results['auc_precision_recall'],
                       initial_results['auc_precision_recall'])
    self.assertGreater(final_results['accuracy'], initial_results['accuracy'])
125
126
127
128

  def test_wide_deep_estimator_training(self):
    self.build_and_test_estimator('wide_deep')

129
130
  def test_end_to_end_wide(self):
    integration.run_synthetic(
131
132
        main=census_main.main, tmp_root=self.get_temp_dir(),
        extra_flags=[
133
134
            '--data_dir', self.get_temp_dir(),
            '--model_type', 'wide',
135
            '--download_if_missing=false'
136
137
138
139
140
        ],
        synth=False, max_train=None)

  def test_end_to_end_deep(self):
    integration.run_synthetic(
141
142
        main=census_main.main, tmp_root=self.get_temp_dir(),
        extra_flags=[
143
144
            '--data_dir', self.get_temp_dir(),
            '--model_type', 'deep',
145
            '--download_if_missing=false'
146
147
148
149
150
        ],
        synth=False, max_train=None)

  def test_end_to_end_wide_deep(self):
    integration.run_synthetic(
151
152
        main=census_main.main, tmp_root=self.get_temp_dir(),
        extra_flags=[
153
154
            '--data_dir', self.get_temp_dir(),
            '--model_type', 'wide_deep',
155
            '--download_if_missing=false'
156
157
158
        ],
        synth=False, max_train=None)

159
160
161

if __name__ == '__main__':
  tf.test.main()