Merge branch 'master' of https://github.com/tensorflow/models into RTESuperGLUE

official/recommendation/neumf_model.py

-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ==============================================================================
+
 """Defines NeuMF model for NCF framework.
 
 Some abbreviations used in the code base:

official/recommendation/popen_helper.py

-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ==============================================================================
+
 """Helper file for running the async data generation process in OSS."""
 
 import contextlib

official/recommendation/stat_utils.py

-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,15 +11,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ==============================================================================
+
 """Statistics utility functions of NCF."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import os
-
 import numpy as np
 
 
@@ -37,7 +35,8 @@ def permutation(args):
     args: A size two tuple that will unpacked into the size of the permutation
       and the random seed. This form is used because starmap is not universally
       available.
-  returns: A NumPy array containing a random permutation.
+  Returns:
+    A NumPy array containing a random permutation.
   """
   x, seed = args
 

official/requirements.txt

@@ -12,7 +12,7 @@ tensorflow-hub>=0.6.0
 tensorflow-model-optimization>=0.4.1
 tensorflow-datasets
 tensorflow-addons
-dataclasses
+dataclasses;python_version<"3.7"
 gin-config
 tf_slim>=1.1.0
 Cython

official/utils/__init__.py

+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+

official/utils/docs/build_api_docs_lib.py

+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+r"""Common library for API docs builder."""
+
+import tensorflow as tf
+from tensorflow_docs.api_generator import doc_controls
+
+
+def hide_module_model_and_layer_methods():
+  """Hide methods and properties defined in the base classes of Keras layers.
+
+  We hide all methods and properties of the base classes, except:
+  - `__init__` is always documented.
+  - `call` is always documented, as it can carry important information for
+    complex layers.
+  """
+  module_contents = list(tf.Module.__dict__.items())
+  model_contents = list(tf.keras.Model.__dict__.items())
+  layer_contents = list(tf.keras.layers.Layer.__dict__.items())
+
+  for name, obj in module_contents + layer_contents + model_contents:
+    if name == '__init__':
+      # Always document __init__.
+      continue
+
+    if name == 'call':
+      # Always document `call`.
+      if hasattr(obj, doc_controls._FOR_SUBCLASS_IMPLEMENTERS):  # pylint: disable=protected-access
+        delattr(obj, doc_controls._FOR_SUBCLASS_IMPLEMENTERS)  # pylint: disable=protected-access
+      continue
+
+    # Otherwise, exclude from documentation.
+    if isinstance(obj, property):
+      obj = obj.fget
+
+    if isinstance(obj, (staticmethod, classmethod)):
+      obj = obj.__func__
+
+    try:
+      doc_controls.do_not_doc_in_subclasses(obj)
+    except AttributeError:
+      pass

official/utils/docs/build_docs.py → official/utils/docs/build_nlp_api_docs.py

-# Lint as: python3
 # Copyright 2021 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -12,16 +11,14 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ==============================================================================
+
 r"""Tool to generate api_docs for tensorflow_models/official library.
 
 Example:
 
 $> pip install -U git+https://github.com/tensorflow/docs
-$> python build_docs \
- --output_dir=/tmp/api_docs \
- --project_short_name=tfnlp \
- --project_full_name="TensorFlow Official Models - NLP Modeling Library"
+$> python build_nlp_api_docs \
+ --output_dir=/tmp/api_docs
 """
 
 import os
@@ -29,13 +26,11 @@ import os
 from absl import app
 from absl import flags
 from absl import logging
-
-import tensorflow as tf
-from tensorflow_docs.api_generator import doc_controls
 from tensorflow_docs.api_generator import generate_lib
 from tensorflow_docs.api_generator import public_api
 
 from official.nlp import modeling as tfnlp
+import build_api_docs_lib
 
 FLAGS = flags.FLAGS
 
@@ -59,46 +54,12 @@ PROJECT_SHORT_NAME = 'tfnlp'
 PROJECT_FULL_NAME = 'TensorFlow Official Models - NLP Modeling Library'
 
 
-def _hide_module_model_and_layer_methods():
-  """Hide methods and properties defined in the base classes of Keras layers.
-
-  We hide all methods and properties of the base classes, except:
-  - `__init__` is always documented.
-  - `call` is always documented, as it can carry important information for
-    complex layers.
-  """
-  module_contents = list(tf.Module.__dict__.items())
-  model_contents = list(tf.keras.Model.__dict__.items())
-  layer_contents = list(tf.keras.layers.Layer.__dict__.items())
-
-  for name, obj in module_contents + layer_contents + model_contents:
-    if name == '__init__':
-      # Always document __init__.
-      continue
-
-    if name == 'call':
-      # Always document `call`.
-      if hasattr(obj, doc_controls._FOR_SUBCLASS_IMPLEMENTERS):  # pylint: disable=protected-access
-        delattr(obj, doc_controls._FOR_SUBCLASS_IMPLEMENTERS)  # pylint: disable=protected-access
-      continue
-
-    # Otherwise, exclude from documentation.
-    if isinstance(obj, property):
-      obj = obj.fget
-
-    if isinstance(obj, (staticmethod, classmethod)):
-      obj = obj.__func__
-
-    try:
-      doc_controls.do_not_doc_in_subclasses(obj)
-    except AttributeError:
-      pass
-
-
 def gen_api_docs(code_url_prefix, site_path, output_dir, gen_report,
                  project_short_name, project_full_name, search_hints):
   """Generates api docs for the tensorflow docs package."""
-  _hide_module_model_and_layer_methods()
+  build_api_docs_lib.hide_module_model_and_layer_methods()
+  del tfnlp.layers.MultiHeadAttention
+  del tfnlp.layers.EinsumDense
 
   doc_generator = generate_lib.DocGenerator(
       root_title=project_full_name,

official/utils/docs/build_vision_api_docs.py

+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+r"""Tool to generate api_docs for tensorflow_models/official library.
+
+Example:
+
+$> pip install -U git+https://github.com/tensorflow/docs
+$> python build_vision_api_docs \
+ --output_dir=/tmp/api_docs
+"""
+
+import os
+
+from absl import app
+from absl import flags
+from absl import logging
+from tensorflow_docs.api_generator import generate_lib
+from tensorflow_docs.api_generator import public_api
+
+import build_api_docs_lib
+from official.vision.beta import modeling as tfvision
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string('output_dir', None, 'Where to write the resulting docs to.')
+flags.DEFINE_string(
+    'code_url_prefix',
+    'https://github.com/tensorflow/models/blob/master/official/vision/beta/modeling/',
+    'The url prefix for links to code.')
+
+flags.DEFINE_bool('search_hints', True,
+                  'Include metadata search hints in the generated files')
+
+flags.DEFINE_string('site_path', 'tfvision/api_docs/python',
+                    'Path prefix in the _toc.yaml')
+
+flags.DEFINE_bool('gen_report', False,
+                  'Generate an API report containing the health of the '
+                  'docstrings of the public API.')
+
+PROJECT_SHORT_NAME = 'tfvision'
+PROJECT_FULL_NAME = 'TensorFlow Official Models - Vision Modeling Library'
+
+
+def gen_api_docs(code_url_prefix, site_path, output_dir, gen_report,
+                 project_short_name, project_full_name, search_hints):
+  """Generates api docs for the tensorflow docs package."""
+  build_api_docs_lib.hide_module_model_and_layer_methods()
+
+  doc_generator = generate_lib.DocGenerator(
+      root_title=project_full_name,
+      py_modules=[(project_short_name, tfvision)],
+      base_dir=os.path.dirname(tfvision.__file__),
+      code_url_prefix=code_url_prefix,
+      search_hints=search_hints,
+      site_path=site_path,
+      gen_report=gen_report,
+      callbacks=[public_api.explicit_package_contents_filter],
+  )
+
+  doc_generator.build(output_dir)
+  logging.info('Output docs to: %s', output_dir)
+
+
+def main(argv):
+  if len(argv) > 1:
+    raise app.UsageError('Too many command-line arguments.')
+
+  gen_api_docs(
+      code_url_prefix=FLAGS.code_url_prefix,
+      site_path=FLAGS.site_path,
+      output_dir=FLAGS.output_dir,
+      gen_report=FLAGS.gen_report,
+      project_short_name=PROJECT_SHORT_NAME,
+      project_full_name=PROJECT_FULL_NAME,
+      search_hints=FLAGS.search_hints)
+
+
+if __name__ == '__main__':
+  flags.mark_flag_as_required('output_dir')
+  app.run(main)

official/utils/flags/__init__.py

+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+

official/utils/flags/_base.py

-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,12 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ==============================================================================
-"""Flags which will be nearly universal across models."""
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
+"""Flags which will be nearly universal across models."""
 
 from absl import flags
 import tensorflow as tf

official/utils/flags/_benchmark.py

-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,12 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ==============================================================================
-"""Flags for benchmarking models."""
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
+"""Flags for benchmarking models."""
 
 from absl import flags
 

official/utils/flags/_conventions.py

-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,12 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ==============================================================================
-"""Central location for shared argparse convention definitions."""
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
+"""Central location for shared argparse convention definitions."""
 
 import sys
 import codecs

official/utils/flags/_device.py

-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,12 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ==============================================================================
-"""Flags for managing compute devices. Currently only contains TPU flags."""
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
+"""Flags for managing compute devices. Currently only contains TPU flags."""
 
 from absl import flags
 from absl import logging

official/utils/flags/_distribution.py

-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,12 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ==============================================================================
-"""Flags related to distributed execution."""
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
+"""Flags related to distributed execution."""
 
 from absl import flags
 import tensorflow as tf

official/utils/flags/_misc.py

-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,12 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ==============================================================================
-"""Misc flags."""
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
+"""Misc flags."""
 
 from absl import flags
 

official/utils/flags/_performance.py

-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,12 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ==============================================================================
-"""Register flags for optimizing performance."""
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
+"""Register flags for optimizing performance."""
 
 import multiprocessing
 

official/utils/flags/core.py

-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,16 +11,12 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ==============================================================================
+
 """Public interface for flag definition.
 
 See _example.py for detailed instructions on defining flags.
 """
 
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
 import sys
 
 from six.moves import shlex_quote

official/utils/flags/flags_test.py

-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ==============================================================================
 
 import unittest
 

official/utils/hyperparams_flags.py

-# Copyright 2019 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,13 +11,8 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-# ==============================================================================
-"""Common flags for importing hyperparameters."""
 
-from __future__ import absolute_import
-from __future__ import division
-# from __future__ import google_type_annotations
-from __future__ import print_function
+"""Common flags for importing hyperparameters."""
 
 from absl import flags
 from official.utils.flags import core as flags_core

official/utils/misc/__init__.py

+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+