Merge branch 'panoptic-segmentation' into panoptic-deeplab-modeling

official/vision/beta/projects/movinet/modeling/movinet.py → official/projects/movinet/modeling/movinet.py

@@ -24,8 +24,8 @@ from typing import Dict, Mapping, Optional, Sequence, Tuple, Union
 import tensorflow as tf
 
 from official.modeling import hyperparams
+from official.projects.movinet.modeling import movinet_layers
 from official.vision.beta.modeling.backbones import factory
-from official.vision.beta.projects.movinet.modeling import movinet_layers
 
 # Defines a set of kernel sizes and stride sizes to simplify and shorten
 # architecture definitions for configs below.
@@ -474,7 +474,7 @@ class Movinet(tf.keras.Model):
         if not (len(block.expand_filters) == len(block.kernel_sizes) ==
                 len(block.strides)):
           raise ValueError(
-              'Lenths of block parameters differ: {}, {}, {}'.format(
+              'Lengths of block parameters differ: {}, {}, {}'.format(
                   len(block.expand_filters),
                   len(block.kernel_sizes),
                   len(block.strides)))
@@ -709,8 +709,12 @@ def build_movinet(
   """Builds MoViNet backbone from a config."""
   backbone_type = backbone_config.type
   backbone_cfg = backbone_config.get()
-  assert backbone_type == 'movinet', ('Inconsistent backbone type '
-                                      f'{backbone_type}')
+  if backbone_type != 'movinet':
+    raise ValueError(f'Inconsistent backbone type {backbone_type}')
+  if norm_activation_config.activation is not None:
+    raise ValueError(
+        'norm_activation is not used in MoViNets, but specified: %s' %
+        norm_activation_config.activation)
 
   return Movinet(
       model_id=backbone_cfg.model_id,

official/vision/beta/projects/movinet/modeling/movinet_layers.py → official/projects/movinet/modeling/movinet_layers.py

@@ -22,6 +22,7 @@ from typing import Any, Mapping, Optional, Sequence, Tuple, Union
 
 import tensorflow as tf
 
+from official.modeling import tf_utils
 from official.vision.beta.modeling.layers import nn_layers
 
 # Default kernel weight decay that may be overridden
@@ -323,7 +324,8 @@ class ConvBlock(tf.keras.layers.Layer):
     self._use_buffered_input = use_buffered_input
 
     if activation is not None:
-      self._activation_layer = tf.keras.layers.Activation(activation)
+      self._activation_layer = tf_utils.get_activation(
+          activation, use_keras_layer=True)
     else:
       self._activation_layer = None
 

official/vision/beta/projects/movinet/modeling/movinet_layers_test.py → official/projects/movinet/modeling/movinet_layers_test.py

@@ -18,8 +18,8 @@
 from absl.testing import parameterized
 import tensorflow as tf
 
+from official.projects.movinet.modeling import movinet_layers
 from official.vision.beta.modeling.layers import nn_layers
-from official.vision.beta.projects.movinet.modeling import movinet_layers
 
 
 class MovinetLayersTest(parameterized.TestCase, tf.test.TestCase):
@@ -338,7 +338,7 @@ class MovinetLayersTest(parameterized.TestCase, tf.test.TestCase):
       predicted = tf.concat(predicted, axis=1)
 
       self.assertEqual(predicted.shape, expected.shape)
-      self.assertAllClose(predicted, expected)
+      self.assertAllClose(predicted, expected, atol=1e-4)
 
       self.assertAllClose(
           predicted,
@@ -349,7 +349,8 @@ class MovinetLayersTest(parameterized.TestCase, tf.test.TestCase):
             [[[3., 3., 3.]],
              [[3., 3., 3.]]],
             [[[4., 4., 4.]],
-             [[4., 4., 4.]]]]])
+             [[4., 4., 4.]]]]],
+          atol=1e-4)
 
   def test_stream_movinet_block(self):
     block = movinet_layers.MovinetBlock(

official/vision/beta/projects/movinet/modeling/movinet_model.py → official/projects/movinet/modeling/movinet_model.py

@@ -21,10 +21,10 @@ from typing import Any, Dict, Mapping, Optional, Sequence, Tuple, Union
 from absl import logging
 import tensorflow as tf
 
+from official.projects.movinet.configs import movinet as cfg
+from official.projects.movinet.modeling import movinet_layers
 from official.vision.beta.modeling import backbones
 from official.vision.beta.modeling import factory_3d as model_factory
-from official.vision.beta.projects.movinet.configs import movinet as cfg
-from official.vision.beta.projects.movinet.modeling import movinet_layers
 
 
 @tf.keras.utils.register_keras_serializable(package='Vision')

official/vision/beta/projects/movinet/modeling/movinet_model_test.py → official/projects/movinet/modeling/movinet_model_test.py

@@ -19,8 +19,8 @@ from absl.testing import parameterized
 import numpy as np
 import tensorflow as tf
 
-from official.vision.beta.projects.movinet.modeling import movinet
-from official.vision.beta.projects.movinet.modeling import movinet_model
+from official.projects.movinet.modeling import movinet
+from official.projects.movinet.modeling import movinet_model
 
 
 class MovinetModelTest(parameterized.TestCase, tf.test.TestCase):

official/vision/beta/projects/movinet/modeling/movinet_test.py → official/projects/movinet/modeling/movinet_test.py

@@ -18,7 +18,7 @@
 from absl.testing import parameterized
 import tensorflow as tf
 
-from official.vision.beta.projects.movinet.modeling import movinet
+from official.projects.movinet.modeling import movinet
 
 
 class MoViNetTest(parameterized.TestCase, tf.test.TestCase):

official/vision/beta/projects/movinet/movinet_tutorial.ipynb → official/projects/movinet/movinet_tutorial.ipynb

@@ -66,10 +66,10 @@
         "import tensorflow_hub as hub\n",
         "\n",
         "from official.vision.beta.configs import video_classification\n",
-        "from official.vision.beta.projects.movinet.configs import movinet as movinet_configs\n",
-        "from official.vision.beta.projects.movinet.modeling import movinet\n",
-        "from official.vision.beta.projects.movinet.modeling import movinet_layers\n",
-        "from official.vision.beta.projects.movinet.modeling import movinet_model"
+        "from official.projects.movinet.configs import movinet as movinet_configs\n",
+        "from official.projects.movinet.modeling import movinet\n",
+        "from official.projects.movinet.modeling import movinet_layers\n",
+        "from official.projects.movinet.modeling import movinet_model"
       ]
     },
     {

official/vision/beta/projects/movinet/tools/convert_3d_2plus1d.py → official/projects/movinet/tools/convert_3d_2plus1d.py

@@ -18,8 +18,8 @@ from absl import app
 from absl import flags
 import tensorflow as tf
 
-from official.vision.beta.projects.movinet.modeling import movinet
-from official.vision.beta.projects.movinet.modeling import movinet_model
+from official.projects.movinet.modeling import movinet
+from official.projects.movinet.modeling import movinet_model
 
 flags.DEFINE_string(
     'input_checkpoint_path', None,
@@ -29,8 +29,10 @@ flags.DEFINE_string(
     'Export path to save the saved_model file.')
 flags.DEFINE_string(
     'model_id', 'a0', 'MoViNet model name.')
+flags.DEFINE_string(
+    'se_type', '2plus3d', 'MoViNet model SE type.')
 flags.DEFINE_bool(
-    'causal', False, 'Run the model in causal mode.')
+    'causal', True, 'Run the model in causal mode.')
 flags.DEFINE_bool(
     'use_positional_encoding', False,
     'Whether to use positional encoding (only applied when causal=True).')
@@ -47,6 +49,7 @@ def main(_) -> None:
       model_id=FLAGS.model_id,
       causal=FLAGS.causal,
       conv_type='2plus1d',
+      se_type=FLAGS.se_type,
       use_positional_encoding=FLAGS.use_positional_encoding)
   model_2plus1d = movinet_model.MovinetClassifier(
       backbone=backbone_2plus1d,
@@ -57,6 +60,7 @@ def main(_) -> None:
       model_id=FLAGS.model_id,
       causal=FLAGS.causal,
       conv_type='3d_2plus1d',
+      se_type=FLAGS.se_type,
       use_positional_encoding=FLAGS.use_positional_encoding)
   model_3d_2plus1d = movinet_model.MovinetClassifier(
       backbone=backbone_3d_2plus1d,

official/vision/beta/projects/movinet/tools/convert_3d_2plus1d_test.py → official/projects/movinet/tools/convert_3d_2plus1d_test.py

@@ -19,9 +19,9 @@ import os
 from absl import flags
 import tensorflow as tf
 
-from official.vision.beta.projects.movinet.modeling import movinet
-from official.vision.beta.projects.movinet.modeling import movinet_model
-from official.vision.beta.projects.movinet.tools import convert_3d_2plus1d
+from official.projects.movinet.modeling import movinet
+from official.projects.movinet.modeling import movinet_model
+from official.projects.movinet.tools import convert_3d_2plus1d
 
 FLAGS = flags.FLAGS
 
@@ -36,7 +36,8 @@ class Convert3d2plus1dTest(tf.test.TestCase):
     model_3d_2plus1d = movinet_model.MovinetClassifier(
         backbone=movinet.Movinet(
             model_id='a0',
-            conv_type='3d_2plus1d'),
+            conv_type='3d_2plus1d',
+            se_type='2plus3d'),
         num_classes=600)
     model_3d_2plus1d.build([1, 1, 1, 1, 3])
     save_checkpoint = tf.train.Checkpoint(model=model_3d_2plus1d)

official/vision/beta/projects/movinet/train.py → official/projects/movinet/train.py

@@ -17,8 +17,8 @@ r"""Training driver.
 
 To train:
 
-CONFIG_FILE=official/vision/beta/projects/movinet/configs/yaml/movinet_a0_k600_8x8.yaml
-python3 official/vision/beta/projects/movinet/train.py \
+CONFIG_FILE=official/projects/movinet/configs/yaml/movinet_a0_k600_8x8.yaml
+python3 official/projects/movinet/train.py \
     --experiment=movinet_kinetics600 \
     --mode=train \
     --model_dir=/tmp/movinet/ \
@@ -47,8 +47,8 @@ from official.modeling import performance
 # model garden factory.
 # pylint: disable=unused-import
 # the followings are the necessary imports.
-from official.vision.beta.projects.movinet.modeling import movinet
-from official.vision.beta.projects.movinet.modeling import movinet_model
+from official.projects.movinet.modeling import movinet
+from official.projects.movinet.modeling import movinet_model
 # pylint: enable=unused-import
 
 FLAGS = flags.FLAGS

official/vision/beta/projects/movinet/train_test.py → official/projects/movinet/train_test.py

@@ -24,8 +24,8 @@ from absl import logging
 from absl.testing import flagsaver
 import tensorflow as tf
 
+from official.projects.movinet import train as train_lib
 from official.vision.beta.dataloaders import tfexample_utils
-from official.vision.beta.projects.movinet import train as train_lib
 
 FLAGS = flags.FLAGS
 

official/projects/nhnet/README.md

@@ -36,7 +36,7 @@ will crawl and extract news articles on a local machine.
 
 First, install the `news-please` CLI (requires python 3.x)
 ```shell
-$ pip3 install news-please
+$ pip3 install news-please==1.4.26
 ```
 
 Next, run the crawler with our provided [config and URL list](https://github.com/google-research-datasets/NewSHead/releases)

official/projects/nhnet/decoder.py

@@ -15,9 +15,10 @@
 """Transformer decoder that mimics a BERT encoder, to load BERT checkpoints."""
 
 import tensorflow as tf
+
+from official.legacy.transformer import model_utils as transformer_utils
 from official.modeling import tf_utils
 from official.nlp.modeling import layers
-from official.nlp.transformer import model_utils as transformer_utils
 
 
 class TransformerDecoder(tf.keras.layers.Layer):

official/projects/nhnet/evaluation.py

@@ -22,8 +22,8 @@ from absl import logging
 import numpy as np
 import tensorflow as tf
 
-from official.nlp.transformer import metrics as metrics_v2
-from official.nlp.transformer.utils import metrics
+from official.legacy.transformer import metrics as metrics_v2
+from official.legacy.transformer.utils import metrics
 from official.projects.nhnet import input_pipeline
 from official.projects.nhnet import models
 

official/projects/nhnet/trainer.py

@@ -25,8 +25,8 @@ from six.moves import zip
 import tensorflow as tf
 
 from official.common import distribute_utils
+from official.legacy.transformer import metrics as transformer_metrics
 from official.modeling.hyperparams import params_dict
-from official.nlp.transformer import metrics as transformer_metrics
 from official.projects.nhnet import evaluation
 from official.projects.nhnet import input_pipeline
 from official.projects.nhnet import models

official/projects/roformer/README.md

+Code for Roformer.
+
+Run with
+```bash
+DATA_PATH=???
+OUTPUT_DIR=???
+python3 train.py \
+  --experiment=roformer/pretraining \
+  --config_file=experiments/roformer_base.yaml \
+  --params_override="task.validation_data.input_path=${DATA_PATH},runtime.distribution_strategy=tpu" \
+  --tpu=local \
+  --model_dir=${OUTPUT_DIR} \
+  --mode=train_and_eval
+```

official/projects/roformer/__init__.py

+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+

official/projects/roformer/experiments/roformer_base.yaml

+task:
+  init_checkpoint: ''
+  model:
+    cls_heads: [{activation: tanh, cls_token_idx: 0, dropout_rate: 0.1, inner_dim: 768, name: next_sentence, num_classes: 2}]
+    encoder:
+      type: any
+      any:
+        attention_dropout_rate: 0.1
+        dropout_rate: 0.1
+        embedding_size: 768
+        hidden_activation: gelu
+        hidden_size: 768
+        initializer_range: 0.02
+        intermediate_size: 3072
+        max_position_embeddings: 512
+        num_attention_heads: 12
+        num_layers: 12
+        type_vocab_size: 2
+        vocab_size: 30522
+  train_data:
+    drop_remainder: true
+    global_batch_size: 256
+    input_path: ''
+    is_training: true
+    max_predictions_per_seq: 76
+    seq_length: 512
+    use_next_sentence_label: true
+    use_position_id: false
+  validation_data:
+    drop_remainder: false
+    global_batch_size: 256
+    input_path: ''
+    is_training: false
+    max_predictions_per_seq: 76
+    seq_length: 512
+    use_next_sentence_label: true
+    use_position_id: false
+trainer:
+  checkpoint_interval: 20000
+  max_to_keep: 5
+  optimizer_config:
+    learning_rate:
+      polynomial:
+        cycle: false
+        decay_steps: 1000000
+        end_learning_rate: 0.0
+        initial_learning_rate: 0.0001
+        power: 1.0
+      type: polynomial
+    optimizer:
+      type: adamw
+    warmup:
+      polynomial:
+        power: 1
+        warmup_steps: 10000
+      type: polynomial
+  steps_per_loop: 50
+  summary_interval: 50
+  train_steps: 1000000
+  validation_interval: 1000
+  validation_steps: 64

official/projects/roformer/roformer.py

+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Roformer model configurations and instantiation methods."""
+import tensorflow as tf
+
+from official.modeling import tf_utils
+from official.modeling.hyperparams import base_config
+from official.nlp.configs import encoders
+from official.projects.roformer import roformer_encoder
+
+
+class RoformerEncoderConfig(encoders.BertEncoderConfig):
+  pass
+
+
+@base_config.bind(RoformerEncoderConfig)
+def get_encoder(encoder_cfg: RoformerEncoderConfig):
+  """Gets a 'RoformerEncoder' object.
+
+  Args:
+    encoder_cfg: A 'RoformerEncoderConfig'.
+
+  Returns:
+    A encoder object.
+  """
+  return roformer_encoder.RoformerEncoder(
+      vocab_size=encoder_cfg.vocab_size,
+      hidden_size=encoder_cfg.hidden_size,
+      num_layers=encoder_cfg.num_layers,
+      num_attention_heads=encoder_cfg.num_attention_heads,
+      intermediate_size=encoder_cfg.intermediate_size,
+      activation=tf_utils.get_activation(encoder_cfg.hidden_activation),
+      dropout_rate=encoder_cfg.dropout_rate,
+      attention_dropout_rate=encoder_cfg.attention_dropout_rate,
+      max_sequence_length=encoder_cfg.max_position_embeddings,
+      type_vocab_size=encoder_cfg.type_vocab_size,
+      initializer=tf.keras.initializers.TruncatedNormal(
+          stddev=encoder_cfg.initializer_range),
+      output_range=encoder_cfg.output_range,
+      embedding_width=encoder_cfg.embedding_size,
+      norm_first=encoder_cfg.norm_first)