Copybara import of the project:

--
0745859aa2300d03f5c9f026c4759a6314e8e14e by Srihari Humbarwadi <sriharihumbarwadi97@gmail.com>:

Added serving module for Panoptic MaskRCNN (#10281)

* added `PanopticSegmentationModule`

* added tests for `PanopticSegmentationModule`

* added script to export `saved_model`

* added registry_imports

* added training script

* fixed docstrings

* pass model as an arg to serving module

COPYBARA_INTEGRATE_REVIEW=https://github.com/tensorflow/models/pull/10283 from tensorflow:panoptic-segmentation 0745859aa2300d03f5c9f026c4759a6314e8e14e
PiperOrigin-RevId: 400837838

official/vision/beta/projects/panoptic_maskrcnn/serving/export_saved_model.py

+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+r"""Panoptic MaskRCNN model export binary for serving/inference.
+
+To export a trained checkpoint in saved_model format (shell script):
+
+CHECKPOINT_PATH = XX
+EXPORT_DIR_PATH = XX
+CONFIG_FILE_PATH = XX
+export_saved_model --export_dir=${EXPORT_DIR_PATH}/ \
+                   --checkpoint_path=${CHECKPOINT_PATH} \
+                   --config_file=${CONFIG_FILE_PATH} \
+                   --batch_size=2 \
+                   --input_image_size=224,224
+To serve (python):
+export_dir_path = XX
+input_type = XX
+input_images = XX
+imported = tf.saved_model.load(export_dir_path)
+model_fn = imported.signatures['serving_default']
+output = model_fn(input_images)
+"""
+
+from absl import app
+from absl import flags
+import tensorflow as tf
+
+from official.core import exp_factory
+from official.modeling import hyperparams
+from official.vision.beta.projects.panoptic_maskrcnn.configs import panoptic_maskrcnn as cfg  # pylint: disable=unused-import
+from official.vision.beta.projects.panoptic_maskrcnn.modeling import factory
+from official.vision.beta.projects.panoptic_maskrcnn.serving import panoptic_segmentation
+from official.vision.beta.projects.panoptic_maskrcnn.tasks import panoptic_maskrcnn as task  # pylint: disable=unused-import
+from official.vision.beta.serving import export_saved_model_lib
+
+FLAGS = flags.FLAGS
+
+flags.DEFINE_string('experiment', 'panoptic_maskrcnn_resnetfpn_coco',
+                    'experiment type, e.g. panoptic_maskrcnn_resnetfpn_coco')
+flags.DEFINE_string('export_dir', None, 'The export directory.')
+flags.DEFINE_string('checkpoint_path', None, 'Checkpoint path.')
+flags.DEFINE_multi_string(
+    'config_file',
+    default=None,
+    help='YAML/JSON files which specifies overrides. The override order '
+    'follows the order of args. Note that each file '
+    'can be used as an override template to override the default parameters '
+    'specified in Python. If the same parameter is specified in both '
+    '`--config_file` and `--params_override`, `config_file` will be used '
+    'first, followed by params_override.')
+flags.DEFINE_string(
+    'params_override', '',
+    'The JSON/YAML file or string which specifies the parameter to be overriden'
+    ' on top of `config_file` template.')
+flags.DEFINE_integer('batch_size', None, 'The batch size.')
+flags.DEFINE_string('input_type', 'image_tensor',
+                    'One of `image_tensor`, `image_bytes`, `tf_example`.')
+flags.DEFINE_string(
+    'input_image_size', '224,224',
+    'The comma-separated string of two integers representing the height,width '
+    'of the input to the model.')
+
+
+def main(_):
+
+  params = exp_factory.get_exp_config(FLAGS.experiment)
+  for config_file in FLAGS.config_file or []:
+    params = hyperparams.override_params_dict(
+        params, config_file, is_strict=True)
+  if FLAGS.params_override:
+    params = hyperparams.override_params_dict(
+        params, FLAGS.params_override, is_strict=True)
+
+  params.validate()
+  params.lock()
+
+  input_image_size = [int(x) for x in FLAGS.input_image_size.split(',')]
+  input_specs = tf.keras.layers.InputSpec(
+      shape=[FLAGS.batch_size, *input_image_size, 3])
+  model = factory.build_panoptic_maskrcnn(
+      input_specs=input_specs, model_config=params.task.model)
+
+  export_module = panoptic_segmentation.PanopticSegmentationModule(
+      params=params,
+      model=model,
+      batch_size=FLAGS.batch_size,
+      input_image_size=[int(x) for x in FLAGS.input_image_size.split(',')],
+      num_channels=3)
+
+  export_saved_model_lib.export_inference_graph(
+      input_type=FLAGS.input_type,
+      batch_size=FLAGS.batch_size,
+      input_image_size=input_image_size,
+      params=params,
+      checkpoint_path=FLAGS.checkpoint_path,
+      export_dir=FLAGS.export_dir,
+      export_module=export_module,
+      export_checkpoint_subdir='checkpoint',
+      export_saved_model_subdir='saved_model')
+
+
+if __name__ == '__main__':
+  app.run(main)

official/vision/beta/projects/panoptic_maskrcnn/serving/panoptic_segmentation.py

+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Panoptic Segmentation input and model functions for serving/inference."""
+
+from typing import List
+
+import tensorflow as tf
+
+from official.modeling.hyperparams import config_definitions as cfg
+from official.vision.beta.projects.panoptic_maskrcnn.modeling import panoptic_maskrcnn_model
+from official.vision.beta.serving import detection
+
+
+class PanopticSegmentationModule(detection.DetectionModule):
+  """Panoptic Segmentation Module."""
+
+  def __init__(self,
+               params: cfg.ExperimentConfig,
+               *,
+               model: tf.keras.Model,
+               batch_size: int,
+               input_image_size: List[int],
+               num_channels: int = 3):
+    """Initializes panoptic segmentation module for export."""
+
+    if batch_size is None:
+      raise ValueError('batch_size cannot be None for panoptic segmentation '
+                       'model.')
+    if not isinstance(model, panoptic_maskrcnn_model.PanopticMaskRCNNModel):
+      raise ValueError('PanopticSegmentationModule module not implemented for '
+                       '{} model.'.format(type(model)))
+
+    super(PanopticSegmentationModule, self).__init__(
+        params=params,
+        model=model,
+        batch_size=batch_size,
+        input_image_size=input_image_size,
+        num_channels=num_channels)
+
+  def serve(self, images: tf.Tensor):
+    """Cast image to float and run inference.
+
+    Args:
+      images: uint8 Tensor of shape [batch_size, None, None, 3]
+    Returns:
+      Tensor holding detection output logits.
+    """
+    model_params = self.params.task.model
+    with tf.device('cpu:0'):
+      images = tf.cast(images, dtype=tf.float32)
+
+      # Tensor Specs for map_fn outputs (images, anchor_boxes, and image_info).
+      images_spec = tf.TensorSpec(shape=self._input_image_size + [3],
+                                  dtype=tf.float32)
+
+      num_anchors = model_params.anchor.num_scales * len(
+          model_params.anchor.aspect_ratios) * 4
+      anchor_shapes = []
+      for level in range(model_params.min_level, model_params.max_level + 1):
+        anchor_level_spec = tf.TensorSpec(
+            shape=[
+                self._input_image_size[0] // 2**level,
+                self._input_image_size[1] // 2**level, num_anchors
+            ],
+            dtype=tf.float32)
+        anchor_shapes.append((str(level), anchor_level_spec))
+
+      image_info_spec = tf.TensorSpec(shape=[4, 2], dtype=tf.float32)
+
+      images, anchor_boxes, image_info = tf.nest.map_structure(
+          tf.identity,
+          tf.map_fn(
+              self._build_inputs,
+              elems=images,
+              fn_output_signature=(images_spec, dict(anchor_shapes),
+                                   image_info_spec),
+              parallel_iterations=32))
+
+    input_image_shape = image_info[:, 1, :]
+
+    # To overcome keras.Model extra limitation to save a model with layers that
+    # have multiple inputs, we use `model.call` here to trigger the forward
+    # path. Note that, this disables some keras magics happens in `__call__`.
+    detections = self.model.call(
+        images=images,
+        image_shape=input_image_shape,
+        anchor_boxes=anchor_boxes,
+        training=False)
+
+    if model_params.detection_generator.apply_nms:
+      final_outputs = {
+          'detection_boxes': detections['detection_boxes'],
+          'detection_scores': detections['detection_scores'],
+          'detection_classes': detections['detection_classes'],
+          'num_detections': detections['num_detections']
+      }
+    else:
+      final_outputs = {
+          'decoded_boxes': detections['decoded_boxes'],
+          'decoded_box_scores': detections['decoded_box_scores']
+      }
+
+    final_outputs.update({
+        'detection_masks': detections['detection_masks'],
+        'segmentation_outputs': detections['segmentation_outputs'],
+        'image_info': image_info
+    })
+    if model_params.generate_panoptic_masks:
+      final_outputs.update({
+          'panoptic_category_mask':
+              detections['panoptic_outputs']['category_mask'],
+          'panoptic_instance_mask':
+              detections['panoptic_outputs']['instance_mask'],
+            })
+
+    return final_outputs

official/vision/beta/projects/panoptic_maskrcnn/serving/panoptic_segmentation_test.py

+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Test for panoptic image segmentation export lib."""
+import io
+import os
+
+from absl.testing import parameterized
+import numpy as np
+from PIL import Image
+import tensorflow as tf
+
+from official.core import exp_factory
+from official.vision.beta.projects.panoptic_maskrcnn.configs import panoptic_maskrcnn as cfg  # pylint: disable=unused-import
+from official.vision.beta.projects.panoptic_maskrcnn.modeling import factory
+from official.vision.beta.projects.panoptic_maskrcnn.serving import panoptic_segmentation
+from official.vision.beta.projects.panoptic_maskrcnn.tasks import panoptic_maskrcnn as task  # pylint: disable=unused-import
+
+
+class PanopticSegmentationExportTest(tf.test.TestCase, parameterized.TestCase):
+
+  def _get_panoptic_segmentation_module(self, experiment_name):
+    params = exp_factory.get_exp_config(experiment_name)
+    params.task.model.backbone.resnet.model_id = 18
+    params.task.model.detection_generator.nms_version = 'batched'
+    input_specs = tf.keras.layers.InputSpec(shape=[1, 128, 128, 3])
+    model = factory.build_panoptic_maskrcnn(
+        input_specs=input_specs, model_config=params.task.model)
+    panoptic_segmentation_module = panoptic_segmentation.PanopticSegmentationModule(
+        params, model=model, batch_size=1, input_image_size=[128, 128])
+    return panoptic_segmentation_module
+
+  def _export_from_module(self, module, input_type, save_directory):
+    signatures = module.get_inference_signatures(
+        {input_type: 'serving_default'})
+    tf.saved_model.save(module, save_directory, signatures=signatures)
+
+  def _get_dummy_input(self, input_type, batch_size, image_size):
+    """Get dummy input for the given input type."""
+    h, w = image_size
+
+    if input_type == 'image_tensor':
+      return tf.zeros((batch_size, h, w, 3), dtype=np.uint8)
+    elif input_type == 'image_bytes':
+      image = Image.fromarray(np.zeros((h, w, 3), dtype=np.uint8))
+      byte_io = io.BytesIO()
+      image.save(byte_io, 'PNG')
+      return [byte_io.getvalue() for b in range(batch_size)]
+    elif input_type == 'tf_example':
+      image_tensor = tf.zeros((h, w, 3), dtype=tf.uint8)
+      encoded_jpeg = tf.image.encode_jpeg(tf.constant(image_tensor)).numpy()
+      example = tf.train.Example(
+          features=tf.train.Features(
+              feature={
+                  'image/encoded':
+                      tf.train.Feature(
+                          bytes_list=tf.train.BytesList(value=[encoded_jpeg])),
+              })).SerializeToString()
+      return [example for b in range(batch_size)]
+
+  @parameterized.parameters(
+      ('image_tensor', 'panoptic_maskrcnn_resnetfpn_coco'),
+      ('image_bytes', 'panoptic_maskrcnn_resnetfpn_coco'),
+      ('tf_example', 'panoptic_maskrcnn_resnetfpn_coco'),
+  )
+  def test_export(self, input_type, experiment_name):
+    tmp_dir = self.get_temp_dir()
+    module = self._get_panoptic_segmentation_module(experiment_name)
+
+    self._export_from_module(module, input_type, tmp_dir)
+
+    self.assertTrue(os.path.exists(os.path.join(tmp_dir, 'saved_model.pb')))
+    self.assertTrue(
+        os.path.exists(os.path.join(tmp_dir, 'variables', 'variables.index')))
+    self.assertTrue(
+        os.path.exists(
+            os.path.join(tmp_dir, 'variables',
+                         'variables.data-00000-of-00001')))
+
+    imported = tf.saved_model.load(tmp_dir)
+    detection_fn = imported.signatures['serving_default']
+
+    images = self._get_dummy_input(
+        input_type, batch_size=1, image_size=[128, 128])
+
+    processed_images, anchor_boxes, image_info = module._build_inputs(
+        tf.zeros((128, 128, 3), dtype=tf.uint8))
+    image_shape = image_info[1, :]
+    image_shape = tf.expand_dims(image_shape, 0)
+    processed_images = tf.expand_dims(processed_images, 0)
+    for l, l_boxes in anchor_boxes.items():
+      anchor_boxes[l] = tf.expand_dims(l_boxes, 0)
+
+    expected_outputs = module.model(
+        images=processed_images,
+        image_shape=image_shape,
+        anchor_boxes=anchor_boxes,
+        training=False)
+    outputs = detection_fn(tf.constant(images))
+
+    self.assertAllClose(outputs['num_detections'].numpy(),
+                        expected_outputs['num_detections'].numpy())
+
+  def test_build_model_fail_with_none_batch_size(self):
+    params = exp_factory.get_exp_config('panoptic_maskrcnn_resnetfpn_coco')
+    input_specs = tf.keras.layers.InputSpec(shape=[1, 128, 128, 3])
+    model = factory.build_panoptic_maskrcnn(
+        input_specs=input_specs, model_config=params.task.model)
+    with self.assertRaisesRegex(
+        ValueError,
+        'batch_size cannot be None for panoptic segmentation model.'):
+      _ = panoptic_segmentation.PanopticSegmentationModule(
+          params, model=model, batch_size=None, input_image_size=[128, 128])
+
+if __name__ == '__main__':
+  tf.test.main()

official/vision/beta/projects/panoptic_maskrcnn/train.py

+# Copyright 2021 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Panoptic MaskRCNN trainer.
+
+All custom registry are imported from registry_imports. Here we use default
+trainer so we directly call train.main. If you need to customize the trainer,
+branch from `official/vision/beta/train.py` and make changes.
+"""
+from absl import app
+
+from official.common import flags as tfm_flags
+from official.vision.beta import train
+from official.vision.beta.projects.panoptic_maskrcnn.configs import panoptic_maskrcnn as cfg  # pylint: disable=unused-import
+from official.vision.beta.projects.panoptic_maskrcnn.tasks import panoptic_maskrcnn as task  # pylint: disable=unused-import
+
+
+if __name__ == '__main__':
+  tfm_flags.define_flags()
+  app.run(train.main)