Add indirection file to NCF async process. (#4958)

* add indirection file

* remove unused imports

* fix import

official/recommendation/data_preprocessing.py

@@ -43,10 +43,7 @@ import tensorflow as tf
 from official.datasets import movielens
 from official.recommendation import constants as rconst
 from official.recommendation import stat_utils
-
-
-_ASYNC_GEN_PATH = os.path.join(os.path.dirname(__file__),
-                               "data_async_generation.py")
+from official.recommendation import popen_helper
 
 
 class NCFDataset(object):
@@ -391,14 +388,11 @@ def instantiate_pipeline(dataset, data_dir, batch_size, eval_batch_size,
   # contention with the main training process.
   subproc_env["CUDA_VISIBLE_DEVICES"] = ""
 
-  python = "python3" if six.PY3 else "python2"
-
   # By limiting the number of workers we guarantee that the worker
   # pool underlying the training generation doesn't starve other processes.
   num_workers = int(multiprocessing.cpu_count() * 0.75)
 
-  subproc_args = [
-      python, _ASYNC_GEN_PATH,
+  subproc_args = popen_helper.INVOCATION + [
       "--data_dir", data_dir,
       "--cache_id", str(ncf_dataset.cache_paths.cache_id),
       "--num_neg", str(num_neg),

official/recommendation/neumf_model.py

@@ -38,10 +38,8 @@ import typing
 
 from six.moves import xrange  # pylint: disable=redefined-builtin
 import tensorflow as tf
-from tensorflow.python.keras.utils import tf_utils
 
 from official.datasets import movielens  # pylint: disable=g-bad-import-order
-from official.utils.accelerator import tpu as tpu_utils
 
 
 def neumf_model_fn(features, labels, mode, params):

official/recommendation/popen_helper.py

+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Helper file for running the async data generation process in OSS."""
+
+import os
+import six
+
+
+_PYTHON = "python3" if six.PY3 else "python2"
+
+_ASYNC_GEN_PATH = os.path.join(os.path.dirname(__file__),
+                               "data_async_generation.py")
+
+INVOCATION = [_PYTHON, _ASYNC_GEN_PATH]