ncf_test.py 7.46 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""Tests NCF."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import math
Reed's avatar
Reed committed
22
import mock
23
24
25
26

import numpy as np
import tensorflow as tf

Reed's avatar
Reed committed
27
from absl.testing import flagsaver
28
from official.recommendation import constants as rconst
29
from official.recommendation import data_pipeline
30
from official.recommendation import neumf_model
Shining Sun's avatar
Shining Sun committed
31
32
from official.recommendation import ncf_common
from official.recommendation import ncf_estimator_main
33
34
35


NUM_TRAIN_NEG = 4
36
37
38


class NcfTest(tf.test.TestCase):
Reed's avatar
Reed committed
39
40
41
42

  @classmethod
  def setUpClass(cls):  # pylint: disable=invalid-name
    super(NcfTest, cls).setUpClass()
Shining Sun's avatar
Shining Sun committed
43
    ncf_common.define_ncf_flags()
Reed's avatar
Reed committed
44

45
46
47
48
49
50
51
52
53
54
55
56
57
  def setUp(self):
    self.top_k_old = rconst.TOP_K
    self.num_eval_negatives_old = rconst.NUM_EVAL_NEGATIVES
    rconst.NUM_EVAL_NEGATIVES = 2

  def tearDown(self):
    rconst.NUM_EVAL_NEGATIVES = self.num_eval_negatives_old
    rconst.TOP_K = self.top_k_old

  def get_hit_rate_and_ndcg(self, predicted_scores_by_user, items_by_user,
                            top_k=rconst.TOP_K, match_mlperf=False):
    rconst.TOP_K = top_k
    rconst.NUM_EVAL_NEGATIVES = predicted_scores_by_user.shape[1] - 1
58
59
60
61
62
63
64
    batch_size = items_by_user.shape[0]

    users = np.repeat(np.arange(batch_size)[:, np.newaxis],
                      rconst.NUM_EVAL_NEGATIVES + 1, axis=1)
    users, items, duplicate_mask = \
      data_pipeline.BaseDataConstructor._assemble_eval_batch(
          users, items_by_user[:, -1:], items_by_user[:, :-1], batch_size)
65
66
67
68
69
70
71

    g = tf.Graph()
    with g.as_default():
      logits = tf.convert_to_tensor(
          predicted_scores_by_user.reshape((-1, 1)), tf.float32)
      softmax_logits = tf.concat([tf.zeros(logits.shape, dtype=logits.dtype),
                                  logits], axis=1)
72
      duplicate_mask = tf.convert_to_tensor(duplicate_mask, tf.float32)
73

Shining Sun's avatar
Shining Sun committed
74
      metric_ops = neumf_model._get_estimator_spec_with_metrics(
75
76
77
78
79
80
81
82
83
84
85
86
87
88
          logits=logits, softmax_logits=softmax_logits,
          duplicate_mask=duplicate_mask, num_training_neg=NUM_TRAIN_NEG,
          match_mlperf=match_mlperf).eval_metric_ops

      hr = metric_ops[rconst.HR_KEY]
      ndcg = metric_ops[rconst.NDCG_KEY]

      init = [tf.global_variables_initializer(),
              tf.local_variables_initializer()]

    with self.test_session(graph=g) as sess:
      sess.run(init)
      return sess.run([hr[1], ndcg[1]])

89
90
91
  def test_hit_rate_and_ndcg(self):
    # Test with no duplicate items
    predictions = np.array([
92
93
94
95
        [2., 0., 1.],  # In top 2
        [1., 0., 2.],  # In top 1
        [2., 1., 0.],  # In top 3
        [3., 4., 2.]   # In top 3
96
97
98
    ])
    items = np.array([
        [2, 3, 1],
99
        [3, 1, 2],
100
        [2, 1, 3],
101
        [1, 3, 2],
102
    ])
103
104

    hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 1)
105
106
    self.assertAlmostEqual(hr, 1 / 4)
    self.assertAlmostEqual(ndcg, 1 / 4)
107
108

    hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 2)
109
110
    self.assertAlmostEqual(hr, 2 / 4)
    self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3)) / 4)
111
112

    hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 3)
113
114
115
116
    self.assertAlmostEqual(hr, 4 / 4)
    self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3) +
                                  2 * math.log(2) / math.log(4)) / 4)

117
118
    hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 1,
                                          match_mlperf=True)
119
120
    self.assertAlmostEqual(hr, 1 / 4)
    self.assertAlmostEqual(ndcg, 1 / 4)
121
122
123

    hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 2,
                                          match_mlperf=True)
124
125
    self.assertAlmostEqual(hr, 2 / 4)
    self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3)) / 4)
126
127
128

    hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 3,
                                          match_mlperf=True)
129
130
131
132
133
134
135
    self.assertAlmostEqual(hr, 4 / 4)
    self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3) +
                                  2 * math.log(2) / math.log(4)) / 4)

    # Test with duplicate items. In the MLPerf case, we treat the duplicates as
    # a single item. Otherwise, we treat the duplicates as separate items.
    predictions = np.array([
136
137
138
139
        [2., 2., 3., 1.],  # In top 4. MLPerf: In top 3
        [1., 0., 2., 3.],  # In top 1. MLPerf: In top 1
        [2., 3., 2., 0.],  # In top 4. MLPerf: In top 3
        [2., 4., 2., 3.]   # In top 2. MLPerf: In top 2
140
141
    ])
    items = np.array([
142
143
144
145
        [2, 2, 3, 1],
        [2, 3, 4, 1],
        [2, 3, 2, 1],
        [3, 2, 1, 4],
146
    ])
147
    hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 1)
148
149
    self.assertAlmostEqual(hr, 1 / 4)
    self.assertAlmostEqual(ndcg, 1 / 4)
150
151

    hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 2)
152
153
    self.assertAlmostEqual(hr, 2 / 4)
    self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3)) / 4)
154
155

    hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 3)
156
157
    self.assertAlmostEqual(hr, 2 / 4)
    self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3)) / 4)
158
159

    hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 4)
160
161
162
163
    self.assertAlmostEqual(hr, 4 / 4)
    self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3) +
                                  2 * math.log(2) / math.log(5)) / 4)

164
165
    hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 1,
                                          match_mlperf=True)
166
167
    self.assertAlmostEqual(hr, 1 / 4)
    self.assertAlmostEqual(ndcg, 1 / 4)
168
169
170

    hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 2,
                                          match_mlperf=True)
171
172
    self.assertAlmostEqual(hr, 2 / 4)
    self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3)) / 4)
173
174
175

    hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 3,
                                          match_mlperf=True)
176
177
178
    self.assertAlmostEqual(hr, 4 / 4)
    self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3) +
                                  2 * math.log(2) / math.log(4)) / 4)
179
180
181

    hr, ndcg = self.get_hit_rate_and_ndcg(predictions, items, 4,
                                          match_mlperf=True)
182
183
184
185
186
    self.assertAlmostEqual(hr, 4 / 4)
    self.assertAlmostEqual(ndcg, (1 + math.log(2) / math.log(3) +
                                  2 * math.log(2) / math.log(4)) / 4)


Reed's avatar
Reed committed
187
188
189
190
191
192
193
  _BASE_END_TO_END_FLAGS = {
      "batch_size": 1024,
      "train_epochs": 1,
      "use_synthetic_data": True
  }

  @flagsaver.flagsaver(**_BASE_END_TO_END_FLAGS)
194
  @mock.patch.object(rconst, "SYNTHETIC_BATCHES_PER_EPOCH", 100)
Reed's avatar
Reed committed
195
  def test_end_to_end(self):
Shining Sun's avatar
Shining Sun committed
196
    ncf_estimator_main.main(None)
Reed's avatar
Reed committed
197
198

  @flagsaver.flagsaver(ml_perf=True, **_BASE_END_TO_END_FLAGS)
199
  @mock.patch.object(rconst, "SYNTHETIC_BATCHES_PER_EPOCH", 100)
Reed's avatar
Reed committed
200
  def test_end_to_end_mlperf(self):
Shining Sun's avatar
Shining Sun committed
201
    ncf_estimator_main.main(None)
Reed's avatar
Reed committed
202

203
204
205
206

if __name__ == "__main__":
  tf.logging.set_verbosity(tf.logging.INFO)
  tf.test.main()