Mask R-CNN model added to models/research/mlperf_object_detection/Mask_RCNN (#4678)

* Create README.md

* readme changed

* readme changed

* ResNet backbone completed.

* FPN added

* Create README.md

* initial commit

* files removed

* initial commit

* protobuf file removed

research/mlperf_object_detection/Mask_RCNN/object_detection/evaluator.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Detection model evaluator.
+
+This file provides a generic evaluation method that can be used to evaluate a
+DetectionModel.
+"""
+
+import logging
+import tensorflow as tf
+
+from object_detection import eval_util
+from object_detection.core import prefetcher
+from object_detection.core import standard_fields as fields
+from object_detection.metrics import coco_evaluation
+from object_detection.utils import object_detection_evaluation
+
+# A dictionary of metric names to classes that implement the metric. The classes
+# in the dictionary must implement
+# utils.object_detection_evaluation.DetectionEvaluator interface.
+EVAL_METRICS_CLASS_DICT = {
+    'pascal_voc_detection_metrics':
+        object_detection_evaluation.PascalDetectionEvaluator,
+    'weighted_pascal_voc_detection_metrics':
+        object_detection_evaluation.WeightedPascalDetectionEvaluator,
+    'pascal_voc_instance_segmentation_metrics':
+        object_detection_evaluation.PascalInstanceSegmentationEvaluator,
+    'weighted_pascal_voc_instance_segmentation_metrics':
+        object_detection_evaluation.WeightedPascalInstanceSegmentationEvaluator,
+    'open_images_V2_detection_metrics':
+        object_detection_evaluation.OpenImagesDetectionEvaluator,
+    'coco_detection_metrics':
+        coco_evaluation.CocoDetectionEvaluator,
+    'coco_mask_metrics':
+        coco_evaluation.CocoMaskEvaluator,
+    'oid_challenge_object_detection_metrics':
+        object_detection_evaluation.OpenImagesDetectionChallengeEvaluator,
+}
+
+EVAL_DEFAULT_METRIC = 'pascal_voc_detection_metrics'
+
+
+def _extract_predictions_and_losses(model,
+                                    create_input_dict_fn,
+                                    ignore_groundtruth=False):
+  """Constructs tensorflow detection graph and returns output tensors.
+
+  Args:
+    model: model to perform predictions with.
+    create_input_dict_fn: function to create input tensor dictionaries.
+    ignore_groundtruth: whether groundtruth should be ignored.
+
+  Returns:
+    prediction_groundtruth_dict: A dictionary with postprocessed tensors (keyed
+      by standard_fields.DetectionResultsFields) and optional groundtruth
+      tensors (keyed by standard_fields.InputDataFields).
+    losses_dict: A dictionary containing detection losses. This is empty when
+      ignore_groundtruth is true.
+  """
+  input_dict = create_input_dict_fn()
+  prefetch_queue = prefetcher.prefetch(input_dict, capacity=500)
+  input_dict = prefetch_queue.dequeue()
+  original_image = tf.expand_dims(input_dict[fields.InputDataFields.image], 0)
+  preprocessed_image, true_image_shapes = model.preprocess(
+      tf.to_float(original_image))
+  prediction_dict = model.predict(preprocessed_image, true_image_shapes)
+  detections = model.postprocess(prediction_dict, true_image_shapes)
+
+  groundtruth = None
+  losses_dict = {}
+  if not ignore_groundtruth:
+    groundtruth = {
+        fields.InputDataFields.groundtruth_boxes:
+            input_dict[fields.InputDataFields.groundtruth_boxes],
+        fields.InputDataFields.groundtruth_classes:
+            input_dict[fields.InputDataFields.groundtruth_classes],
+        fields.InputDataFields.groundtruth_area:
+            input_dict[fields.InputDataFields.groundtruth_area],
+        fields.InputDataFields.groundtruth_is_crowd:
+            input_dict[fields.InputDataFields.groundtruth_is_crowd],
+        fields.InputDataFields.groundtruth_difficult:
+            input_dict[fields.InputDataFields.groundtruth_difficult]
+    }
+    if fields.InputDataFields.groundtruth_group_of in input_dict:
+      groundtruth[fields.InputDataFields.groundtruth_group_of] = (
+          input_dict[fields.InputDataFields.groundtruth_group_of])
+    groundtruth_masks_list = None
+    if fields.DetectionResultFields.detection_masks in detections:
+      groundtruth[fields.InputDataFields.groundtruth_instance_masks] = (
+          input_dict[fields.InputDataFields.groundtruth_instance_masks])
+      groundtruth_masks_list = [
+          input_dict[fields.InputDataFields.groundtruth_instance_masks]]
+    groundtruth_keypoints_list = None
+    if fields.DetectionResultFields.detection_keypoints in detections:
+      groundtruth[fields.InputDataFields.groundtruth_keypoints] = (
+          input_dict[fields.InputDataFields.groundtruth_keypoints])
+      groundtruth_keypoints_list = [
+          input_dict[fields.InputDataFields.groundtruth_keypoints]]
+    label_id_offset = 1
+    model.provide_groundtruth(
+        [input_dict[fields.InputDataFields.groundtruth_boxes]],
+        [tf.one_hot(input_dict[fields.InputDataFields.groundtruth_classes]
+                    - label_id_offset, depth=model.num_classes)],
+        groundtruth_masks_list, groundtruth_keypoints_list)
+    losses_dict.update(model.loss(prediction_dict, true_image_shapes))
+
+  result_dict = eval_util.result_dict_for_single_example(
+      original_image,
+      input_dict[fields.InputDataFields.source_id],
+      detections,
+      groundtruth,
+      class_agnostic=(
+          fields.DetectionResultFields.detection_classes not in detections),
+      scale_to_absolute=True)
+  return result_dict, losses_dict
+
+
+def get_evaluators(eval_config, categories):
+  """Returns the evaluator class according to eval_config, valid for categories.
+
+  Args:
+    eval_config: evaluation configurations.
+    categories: a list of categories to evaluate.
+  Returns:
+    An list of instances of DetectionEvaluator.
+
+  Raises:
+    ValueError: if metric is not in the metric class dictionary.
+  """
+  eval_metric_fn_keys = eval_config.metrics_set
+  if not eval_metric_fn_keys:
+    eval_metric_fn_keys = [EVAL_DEFAULT_METRIC]
+  evaluators_list = []
+  for eval_metric_fn_key in eval_metric_fn_keys:
+    if eval_metric_fn_key not in EVAL_METRICS_CLASS_DICT:
+      raise ValueError('Metric not found: {}'.format(eval_metric_fn_key))
+    evaluators_list.append(
+        EVAL_METRICS_CLASS_DICT[eval_metric_fn_key](categories=categories))
+  return evaluators_list
+
+
+def evaluate(create_input_dict_fn, create_model_fn, eval_config, categories,
+             checkpoint_dir, eval_dir, graph_hook_fn=None, evaluator_list=None):
+  """Evaluation function for detection models.
+
+  Args:
+    create_input_dict_fn: a function to create a tensor input dictionary.
+    create_model_fn: a function that creates a DetectionModel.
+    eval_config: a eval_pb2.EvalConfig protobuf.
+    categories: a list of category dictionaries. Each dict in the list should
+                have an integer 'id' field and string 'name' field.
+    checkpoint_dir: directory to load the checkpoints to evaluate from.
+    eval_dir: directory to write evaluation metrics summary to.
+    graph_hook_fn: Optional function that is called after the training graph is
+      completely built. This is helpful to perform additional changes to the
+      training graph such as optimizing batchnorm. The function should modify
+      the default graph.
+    evaluator_list: Optional list of instances of DetectionEvaluator. If not
+      given, this list of metrics is created according to the eval_config.
+
+  Returns:
+    metrics: A dictionary containing metric names and values from the latest
+      run.
+  """
+  with tf.Graph().as_default():
+    model = create_model_fn()
+
+    if eval_config.ignore_groundtruth and not eval_config.export_path:
+      logging.fatal('If ignore_groundtruth=True then an export_path is '
+                    'required. Aborting!!!')
+
+    tensor_dict, losses_dict = _extract_predictions_and_losses(
+        model=model,
+        create_input_dict_fn=create_input_dict_fn,
+        ignore_groundtruth=eval_config.ignore_groundtruth)
+
+    def _process_batch(tensor_dict, sess, batch_index, counters,
+                       losses_dict=None):
+      """Evaluates tensors in tensor_dict, losses_dict and visualizes examples.
+
+      This function calls sess.run on tensor_dict, evaluating the original_image
+      tensor only on the first K examples and visualizing detections overlaid
+      on this original_image.
+
+      Args:
+        tensor_dict: a dictionary of tensors
+        sess: tensorflow session
+        batch_index: the index of the batch amongst all batches in the run.
+        counters: a dictionary holding 'success' and 'skipped' fields which can
+          be updated to keep track of number of successful and failed runs,
+          respectively.  If these fields are not updated, then the success/skipped
+          counter values shown at the end of evaluation will be incorrect.
+        losses_dict: Optional dictonary of scalar loss tensors.
+
+      Returns:
+        result_dict: a dictionary of numpy arrays
+        result_losses_dict: a dictionary of scalar losses. This is empty if input
+          losses_dict is None.
+      """
+      try:
+        if not losses_dict:
+          losses_dict = {}
+        result_dict, result_losses_dict = sess.run([tensor_dict, losses_dict])
+        counters['success'] += 1
+      except tf.errors.InvalidArgumentError:
+        logging.info('Skipping image')
+        counters['skipped'] += 1
+        return {}, {}
+      global_step = tf.train.global_step(sess, tf.train.get_global_step())
+      if batch_index < eval_config.num_visualizations:
+        tag = 'image-{}'.format(batch_index)
+        eval_util.visualize_detection_results(
+            result_dict,
+            tag,
+            global_step,
+            categories=categories,
+            summary_dir=eval_dir,
+            export_dir=eval_config.visualization_export_dir,
+            show_groundtruth=eval_config.visualize_groundtruth_boxes,
+            groundtruth_box_visualization_color=eval_config.
+            groundtruth_box_visualization_color,
+            min_score_thresh=eval_config.min_score_threshold,
+            max_num_predictions=eval_config.max_num_boxes_to_visualize,
+            skip_scores=eval_config.skip_scores,
+            skip_labels=eval_config.skip_labels,
+            keep_image_id_for_visualization_export=eval_config.
+            keep_image_id_for_visualization_export)
+      return result_dict, result_losses_dict
+
+    variables_to_restore = tf.global_variables()
+    global_step = tf.train.get_or_create_global_step()
+    variables_to_restore.append(global_step)
+
+    if graph_hook_fn: graph_hook_fn()
+
+    if eval_config.use_moving_averages:
+      variable_averages = tf.train.ExponentialMovingAverage(0.0)
+      variables_to_restore = variable_averages.variables_to_restore()
+    saver = tf.train.Saver(variables_to_restore)
+
+    def _restore_latest_checkpoint(sess):
+      latest_checkpoint = tf.train.latest_checkpoint(checkpoint_dir)
+      saver.restore(sess, latest_checkpoint)
+
+    if not evaluator_list:
+      evaluator_list = get_evaluators(eval_config, categories)
+
+    metrics = eval_util.repeated_checkpoint_run(
+        tensor_dict=tensor_dict,
+        summary_dir=eval_dir,
+        evaluators=evaluator_list,
+        batch_processor=_process_batch,
+        checkpoint_dirs=[checkpoint_dir],
+        variables_to_restore=None,
+        restore_fn=_restore_latest_checkpoint,
+        num_batches=eval_config.num_examples,
+        eval_interval_secs=eval_config.eval_interval_secs,
+        max_number_of_evaluations=(1 if eval_config.ignore_groundtruth else
+                                   eval_config.max_evals
+                                   if eval_config.max_evals else None),
+        master=eval_config.eval_master,
+        save_graph=eval_config.save_graph,
+        save_graph_dir=(eval_dir if eval_config.save_graph else ''),
+        losses_dict=losses_dict)
+
+    return metrics

research/mlperf_object_detection/Mask_RCNN/object_detection/export_inference_graph.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+r"""Tool to export an object detection model for inference.
+
+Prepares an object detection tensorflow graph for inference using model
+configuration and an optional trained checkpoint. Outputs inference
+graph, associated checkpoint files, a frozen inference graph and a
+SavedModel (https://tensorflow.github.io/serving/serving_basic.html).
+
+The inference graph contains one of three input nodes depending on the user
+specified option.
+  * `image_tensor`: Accepts a uint8 4-D tensor of shape [None, None, None, 3]
+  * `encoded_image_string_tensor`: Accepts a 1-D string tensor of shape [None]
+    containing encoded PNG or JPEG images. Image resolutions are expected to be
+    the same if more than 1 image is provided.
+  * `tf_example`: Accepts a 1-D string tensor of shape [None] containing
+    serialized TFExample protos. Image resolutions are expected to be the same
+    if more than 1 image is provided.
+
+and the following output nodes returned by the model.postprocess(..):
+  * `num_detections`: Outputs float32 tensors of the form [batch]
+      that specifies the number of valid boxes per image in the batch.
+  * `detection_boxes`: Outputs float32 tensors of the form
+      [batch, num_boxes, 4] containing detected boxes.
+  * `detection_scores`: Outputs float32 tensors of the form
+      [batch, num_boxes] containing class scores for the detections.
+  * `detection_classes`: Outputs float32 tensors of the form
+      [batch, num_boxes] containing classes for the detections.
+  * `detection_masks`: Outputs float32 tensors of the form
+      [batch, num_boxes, mask_height, mask_width] containing predicted instance
+      masks for each box if its present in the dictionary of postprocessed
+      tensors returned by the model.
+
+Notes:
+ * This tool uses `use_moving_averages` from eval_config to decide which
+   weights to freeze.
+
+Example Usage:
+--------------
+python export_inference_graph \
+    --input_type image_tensor \
+    --pipeline_config_path path/to/ssd_inception_v2.config \
+    --trained_checkpoint_prefix path/to/model.ckpt \
+    --output_directory path/to/exported_model_directory
+
+The expected output would be in the directory
+path/to/exported_model_directory (which is created if it does not exist)
+with contents:
+ - graph.pbtxt
+ - model.ckpt.data-00000-of-00001
+ - model.ckpt.info
+ - model.ckpt.meta
+ - frozen_inference_graph.pb
+ + saved_model (a directory)
+
+Config overrides (see the `config_override` flag) are text protobufs
+(also of type pipeline_pb2.TrainEvalPipelineConfig) which are used to override
+certain fields in the provided pipeline_config_path.  These are useful for
+making small changes to the inference graph that differ from the training or
+eval config.
+
+Example Usage (in which we change the second stage post-processing score
+threshold to be 0.5):
+
+python export_inference_graph \
+    --input_type image_tensor \
+    --pipeline_config_path path/to/ssd_inception_v2.config \
+    --trained_checkpoint_prefix path/to/model.ckpt \
+    --output_directory path/to/exported_model_directory \
+    --config_override " \
+            model{ \
+              faster_rcnn { \
+                second_stage_post_processing { \
+                  batch_non_max_suppression { \
+                    score_threshold: 0.5 \
+                  } \
+                } \
+              } \
+            }"
+"""
+import tensorflow as tf
+from google.protobuf import text_format
+from object_detection import exporter
+from object_detection.protos import pipeline_pb2
+
+slim = tf.contrib.slim
+flags = tf.app.flags
+
+flags.DEFINE_string('input_type', 'image_tensor', 'Type of input node. Can be '
+                    'one of [`image_tensor`, `encoded_image_string_tensor`, '
+                    '`tf_example`]')
+flags.DEFINE_string('input_shape', None,
+                    'If input_type is `image_tensor`, this can explicitly set '
+                    'the shape of this input tensor to a fixed size. The '
+                    'dimensions are to be provided as a comma-separated list '
+                    'of integers. A value of -1 can be used for unknown '
+                    'dimensions. If not specified, for an `image_tensor, the '
+                    'default shape will be partially specified as '
+                    '`[None, None, None, 3]`.')
+flags.DEFINE_string('pipeline_config_path', None,
+                    'Path to a pipeline_pb2.TrainEvalPipelineConfig config '
+                    'file.')
+flags.DEFINE_string('trained_checkpoint_prefix', None,
+                    'Path to trained checkpoint, typically of the form '
+                    'path/to/model.ckpt')
+flags.DEFINE_string('output_directory', None, 'Path to write outputs.')
+flags.DEFINE_string('config_override', '',
+                    'pipeline_pb2.TrainEvalPipelineConfig '
+                    'text proto to override pipeline_config_path.')
+tf.app.flags.mark_flag_as_required('pipeline_config_path')
+tf.app.flags.mark_flag_as_required('trained_checkpoint_prefix')
+tf.app.flags.mark_flag_as_required('output_directory')
+FLAGS = flags.FLAGS
+
+
+def main(_):
+  pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+  with tf.gfile.GFile(FLAGS.pipeline_config_path, 'r') as f:
+    text_format.Merge(f.read(), pipeline_config)
+  text_format.Merge(FLAGS.config_override, pipeline_config)
+  if FLAGS.input_shape:
+    input_shape = [
+        int(dim) if dim != '-1' else None
+        for dim in FLAGS.input_shape.split(',')
+    ]
+  else:
+    input_shape = None
+  exporter.export_inference_graph(FLAGS.input_type, pipeline_config,
+                                  FLAGS.trained_checkpoint_prefix,
+                                  FLAGS.output_directory, input_shape)
+
+
+if __name__ == '__main__':
+  tf.app.run()

research/mlperf_object_detection/Mask_RCNN/object_detection/exporter.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Functions to export object detection inference graph."""
+import logging
+import os
+import tempfile
+import tensorflow as tf
+from google.protobuf import text_format
+from tensorflow.core.protobuf import saver_pb2
+from tensorflow.python import pywrap_tensorflow
+from tensorflow.python.client import session
+from tensorflow.python.framework import graph_util
+from tensorflow.python.platform import gfile
+from tensorflow.python.saved_model import signature_constants
+from tensorflow.python.training import saver as saver_lib
+from object_detection.builders import model_builder
+from object_detection.core import standard_fields as fields
+from object_detection.data_decoders import tf_example_decoder
+
+slim = tf.contrib.slim
+
+
+# TODO(derekjchow): Replace with freeze_graph.freeze_graph_with_def_protos when
+# newer version of Tensorflow becomes more common.
+def freeze_graph_with_def_protos(
+    input_graph_def,
+    input_saver_def,
+    input_checkpoint,
+    output_node_names,
+    restore_op_name,
+    filename_tensor_name,
+    clear_devices,
+    initializer_nodes,
+    variable_names_blacklist=''):
+  """Converts all variables in a graph and checkpoint into constants."""
+  del restore_op_name, filename_tensor_name  # Unused by updated loading code.
+
+  # 'input_checkpoint' may be a prefix if we're using Saver V2 format
+  if not saver_lib.checkpoint_exists(input_checkpoint):
+    raise ValueError(
+        'Input checkpoint "' + input_checkpoint + '" does not exist!')
+
+  if not output_node_names:
+    raise ValueError(
+        'You must supply the name of a node to --output_node_names.')
+
+  # Remove all the explicit device specifications for this node. This helps to
+  # make the graph more portable.
+  if clear_devices:
+    for node in input_graph_def.node:
+      node.device = ''
+
+  with tf.Graph().as_default():
+    tf.import_graph_def(input_graph_def, name='')
+    config = tf.ConfigProto(graph_options=tf.GraphOptions())
+    with session.Session(config=config) as sess:
+      if input_saver_def:
+        saver = saver_lib.Saver(saver_def=input_saver_def)
+        saver.restore(sess, input_checkpoint)
+      else:
+        var_list = {}
+        reader = pywrap_tensorflow.NewCheckpointReader(input_checkpoint)
+        var_to_shape_map = reader.get_variable_to_shape_map()
+        for key in var_to_shape_map:
+          try:
+            tensor = sess.graph.get_tensor_by_name(key + ':0')
+          except KeyError:
+            # This tensor doesn't exist in the graph (for example it's
+            # 'global_step' or a similar housekeeping element) so skip it.
+            continue
+          var_list[key] = tensor
+        saver = saver_lib.Saver(var_list=var_list)
+        saver.restore(sess, input_checkpoint)
+        if initializer_nodes:
+          sess.run(initializer_nodes)
+
+      variable_names_blacklist = (variable_names_blacklist.split(',') if
+                                  variable_names_blacklist else None)
+      output_graph_def = graph_util.convert_variables_to_constants(
+          sess,
+          input_graph_def,
+          output_node_names.split(','),
+          variable_names_blacklist=variable_names_blacklist)
+
+  return output_graph_def
+
+
+def replace_variable_values_with_moving_averages(graph,
+                                                 current_checkpoint_file,
+                                                 new_checkpoint_file):
+  """Replaces variable values in the checkpoint with their moving averages.
+
+  If the current checkpoint has shadow variables maintaining moving averages of
+  the variables defined in the graph, this function generates a new checkpoint
+  where the variables contain the values of their moving averages.
+
+  Args:
+    graph: a tf.Graph object.
+    current_checkpoint_file: a checkpoint containing both original variables and
+      their moving averages.
+    new_checkpoint_file: file path to write a new checkpoint.
+  """
+  with graph.as_default():
+    variable_averages = tf.train.ExponentialMovingAverage(0.0)
+    ema_variables_to_restore = variable_averages.variables_to_restore()
+    with tf.Session() as sess:
+      read_saver = tf.train.Saver(ema_variables_to_restore)
+      read_saver.restore(sess, current_checkpoint_file)
+      write_saver = tf.train.Saver()
+      write_saver.save(sess, new_checkpoint_file)
+
+
+def _image_tensor_input_placeholder(input_shape=None):
+  """Returns input placeholder and a 4-D uint8 image tensor."""
+  if input_shape is None:
+    input_shape = (None, None, None, 3)
+  input_tensor = tf.placeholder(
+      dtype=tf.uint8, shape=input_shape, name='image_tensor')
+  return input_tensor, input_tensor
+
+
+def _tf_example_input_placeholder():
+  """Returns input that accepts a batch of strings with tf examples.
+
+  Returns:
+    a tuple of input placeholder and the output decoded images.
+  """
+  batch_tf_example_placeholder = tf.placeholder(
+      tf.string, shape=[None], name='tf_example')
+  def decode(tf_example_string_tensor):
+    tensor_dict = tf_example_decoder.TfExampleDecoder().decode(
+        tf_example_string_tensor)
+    image_tensor = tensor_dict[fields.InputDataFields.image]
+    return image_tensor
+  return (batch_tf_example_placeholder,
+          tf.map_fn(decode,
+                    elems=batch_tf_example_placeholder,
+                    dtype=tf.uint8,
+                    parallel_iterations=32,
+                    back_prop=False))
+
+
+def _encoded_image_string_tensor_input_placeholder():
+  """Returns input that accepts a batch of PNG or JPEG strings.
+
+  Returns:
+    a tuple of input placeholder and the output decoded images.
+  """
+  batch_image_str_placeholder = tf.placeholder(
+      dtype=tf.string,
+      shape=[None],
+      name='encoded_image_string_tensor')
+  def decode(encoded_image_string_tensor):
+    image_tensor = tf.image.decode_image(encoded_image_string_tensor,
+                                         channels=3)
+    image_tensor.set_shape((None, None, 3))
+    return image_tensor
+  return (batch_image_str_placeholder,
+          tf.map_fn(
+              decode,
+              elems=batch_image_str_placeholder,
+              dtype=tf.uint8,
+              parallel_iterations=32,
+              back_prop=False))
+
+
+input_placeholder_fn_map = {
+    'image_tensor': _image_tensor_input_placeholder,
+    'encoded_image_string_tensor':
+    _encoded_image_string_tensor_input_placeholder,
+    'tf_example': _tf_example_input_placeholder,
+}
+
+
+def _add_output_tensor_nodes(postprocessed_tensors,
+                             output_collection_name='inference_op'):
+  """Adds output nodes for detection boxes and scores.
+
+  Adds the following nodes for output tensors -
+    * num_detections: float32 tensor of shape [batch_size].
+    * detection_boxes: float32 tensor of shape [batch_size, num_boxes, 4]
+      containing detected boxes.
+    * detection_scores: float32 tensor of shape [batch_size, num_boxes]
+      containing scores for the detected boxes.
+    * detection_classes: float32 tensor of shape [batch_size, num_boxes]
+      containing class predictions for the detected boxes.
+    * detection_keypoints: (Optional) float32 tensor of shape
+      [batch_size, num_boxes, num_keypoints, 2] containing keypoints for each
+      detection box.
+    * detection_masks: (Optional) float32 tensor of shape
+      [batch_size, num_boxes, mask_height, mask_width] containing masks for each
+      detection box.
+
+  Args:
+    postprocessed_tensors: a dictionary containing the following fields
+      'detection_boxes': [batch, max_detections, 4]
+      'detection_scores': [batch, max_detections]
+      'detection_classes': [batch, max_detections]
+      'detection_masks': [batch, max_detections, mask_height, mask_width]
+        (optional).
+      'num_detections': [batch]
+    output_collection_name: Name of collection to add output tensors to.
+
+  Returns:
+    A tensor dict containing the added output tensor nodes.
+  """
+  detection_fields = fields.DetectionResultFields
+  label_id_offset = 1
+  boxes = postprocessed_tensors.get(detection_fields.detection_boxes)
+  scores = postprocessed_tensors.get(detection_fields.detection_scores)
+  classes = postprocessed_tensors.get(
+      detection_fields.detection_classes) + label_id_offset
+  keypoints = postprocessed_tensors.get(detection_fields.detection_keypoints)
+  masks = postprocessed_tensors.get(detection_fields.detection_masks)
+  num_detections = postprocessed_tensors.get(detection_fields.num_detections)
+  outputs = {}
+  outputs[detection_fields.detection_boxes] = tf.identity(
+      boxes, name=detection_fields.detection_boxes)
+  outputs[detection_fields.detection_scores] = tf.identity(
+      scores, name=detection_fields.detection_scores)
+  outputs[detection_fields.detection_classes] = tf.identity(
+      classes, name=detection_fields.detection_classes)
+  outputs[detection_fields.num_detections] = tf.identity(
+      num_detections, name=detection_fields.num_detections)
+  if keypoints is not None:
+    outputs[detection_fields.detection_keypoints] = tf.identity(
+        keypoints, name=detection_fields.detection_keypoints)
+  if masks is not None:
+    outputs[detection_fields.detection_masks] = tf.identity(
+        masks, name=detection_fields.detection_masks)
+  for output_key in outputs:
+    tf.add_to_collection(output_collection_name, outputs[output_key])
+  if masks is not None:
+    tf.add_to_collection(output_collection_name,
+                         outputs[detection_fields.detection_masks])
+  return outputs
+
+
+def write_frozen_graph(frozen_graph_path, frozen_graph_def):
+  """Writes frozen graph to disk.
+
+  Args:
+    frozen_graph_path: Path to write inference graph.
+    frozen_graph_def: tf.GraphDef holding frozen graph.
+  """
+  with gfile.GFile(frozen_graph_path, 'wb') as f:
+    f.write(frozen_graph_def.SerializeToString())
+  logging.info('%d ops in the final graph.', len(frozen_graph_def.node))
+
+
+def write_saved_model(saved_model_path,
+                      frozen_graph_def,
+                      inputs,
+                      outputs):
+  """Writes SavedModel to disk.
+
+  If checkpoint_path is not None bakes the weights into the graph thereby
+  eliminating the need of checkpoint files during inference. If the model
+  was trained with moving averages, setting use_moving_averages to true
+  restores the moving averages, otherwise the original set of variables
+  is restored.
+
+  Args:
+    saved_model_path: Path to write SavedModel.
+    frozen_graph_def: tf.GraphDef holding frozen graph.
+    inputs: The input image tensor to use for detection.
+    outputs: A tensor dictionary containing the outputs of a DetectionModel.
+  """
+  with tf.Graph().as_default():
+    with session.Session() as sess:
+
+      tf.import_graph_def(frozen_graph_def, name='')
+
+      builder = tf.saved_model.builder.SavedModelBuilder(saved_model_path)
+
+      tensor_info_inputs = {
+          'inputs': tf.saved_model.utils.build_tensor_info(inputs)}
+      tensor_info_outputs = {}
+      for k, v in outputs.items():
+        tensor_info_outputs[k] = tf.saved_model.utils.build_tensor_info(v)
+
+      detection_signature = (
+          tf.saved_model.signature_def_utils.build_signature_def(
+              inputs=tensor_info_inputs,
+              outputs=tensor_info_outputs,
+              method_name=signature_constants.PREDICT_METHOD_NAME))
+
+      builder.add_meta_graph_and_variables(
+          sess, [tf.saved_model.tag_constants.SERVING],
+          signature_def_map={
+              signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
+                  detection_signature,
+          },
+      )
+      builder.save()
+
+
+def write_graph_and_checkpoint(inference_graph_def,
+                               model_path,
+                               input_saver_def,
+                               trained_checkpoint_prefix):
+  """Writes the graph and the checkpoint into disk."""
+  for node in inference_graph_def.node:
+    node.device = ''
+  with tf.Graph().as_default():
+    tf.import_graph_def(inference_graph_def, name='')
+    with session.Session() as sess:
+      saver = saver_lib.Saver(saver_def=input_saver_def,
+                              save_relative_paths=True)
+      saver.restore(sess, trained_checkpoint_prefix)
+      saver.save(sess, model_path)
+
+
+def _get_outputs_from_inputs(input_tensors, detection_model,
+                             output_collection_name):
+  inputs = tf.to_float(input_tensors)
+  preprocessed_inputs, true_image_shapes = detection_model.preprocess(inputs)
+  output_tensors = detection_model.predict(
+      preprocessed_inputs, true_image_shapes)
+  postprocessed_tensors = detection_model.postprocess(
+      output_tensors, true_image_shapes)
+  return _add_output_tensor_nodes(postprocessed_tensors,
+                                  output_collection_name)
+
+
+def _build_detection_graph(input_type, detection_model, input_shape,
+                           output_collection_name, graph_hook_fn):
+  """Build the detection graph."""
+  if input_type not in input_placeholder_fn_map:
+    raise ValueError('Unknown input type: {}'.format(input_type))
+  placeholder_args = {}
+  if input_shape is not None:
+    if input_type != 'image_tensor':
+      raise ValueError('Can only specify input shape for `image_tensor` '
+                       'inputs.')
+    placeholder_args['input_shape'] = input_shape
+  placeholder_tensor, input_tensors = input_placeholder_fn_map[input_type](
+      **placeholder_args)
+  outputs = _get_outputs_from_inputs(
+      input_tensors=input_tensors,
+      detection_model=detection_model,
+      output_collection_name=output_collection_name)
+
+  # Add global step to the graph.
+  slim.get_or_create_global_step()
+
+  if graph_hook_fn: graph_hook_fn()
+
+  return outputs, placeholder_tensor
+
+
+def _export_inference_graph(input_type,
+                            detection_model,
+                            use_moving_averages,
+                            trained_checkpoint_prefix,
+                            output_directory,
+                            additional_output_tensor_names=None,
+                            input_shape=None,
+                            output_collection_name='inference_op',
+                            graph_hook_fn=None):
+  """Export helper."""
+  tf.gfile.MakeDirs(output_directory)
+  frozen_graph_path = os.path.join(output_directory,
+                                   'frozen_inference_graph.pb')
+  saved_model_path = os.path.join(output_directory, 'saved_model')
+  model_path = os.path.join(output_directory, 'model.ckpt')
+
+  outputs, placeholder_tensor = _build_detection_graph(
+      input_type=input_type,
+      detection_model=detection_model,
+      input_shape=input_shape,
+      output_collection_name=output_collection_name,
+      graph_hook_fn=graph_hook_fn)
+
+  saver_kwargs = {}
+  if use_moving_averages:
+    # This check is to be compatible with both version of SaverDef.
+    if os.path.isfile(trained_checkpoint_prefix):
+      saver_kwargs['write_version'] = saver_pb2.SaverDef.V1
+      temp_checkpoint_prefix = tempfile.NamedTemporaryFile().name
+    else:
+      temp_checkpoint_prefix = tempfile.mkdtemp()
+    replace_variable_values_with_moving_averages(
+        tf.get_default_graph(), trained_checkpoint_prefix,
+        temp_checkpoint_prefix)
+    checkpoint_to_use = temp_checkpoint_prefix
+  else:
+    checkpoint_to_use = trained_checkpoint_prefix
+
+  saver = tf.train.Saver(**saver_kwargs)
+  input_saver_def = saver.as_saver_def()
+
+  write_graph_and_checkpoint(
+      inference_graph_def=tf.get_default_graph().as_graph_def(),
+      model_path=model_path,
+      input_saver_def=input_saver_def,
+      trained_checkpoint_prefix=checkpoint_to_use)
+
+  if additional_output_tensor_names is not None:
+    output_node_names = ','.join(outputs.keys()+additional_output_tensor_names)
+  else:
+    output_node_names = ','.join(outputs.keys())
+
+  frozen_graph_def = freeze_graph_with_def_protos(
+      input_graph_def=tf.get_default_graph().as_graph_def(),
+      input_saver_def=input_saver_def,
+      input_checkpoint=checkpoint_to_use,
+      output_node_names=output_node_names,
+      restore_op_name='save/restore_all',
+      filename_tensor_name='save/Const:0',
+      clear_devices=True,
+      initializer_nodes='')
+  write_frozen_graph(frozen_graph_path, frozen_graph_def)
+  write_saved_model(saved_model_path, frozen_graph_def,
+                    placeholder_tensor, outputs)
+
+
+def export_inference_graph(input_type,
+                           pipeline_config,
+                           trained_checkpoint_prefix,
+                           output_directory,
+                           input_shape=None,
+                           output_collection_name='inference_op',
+                           additional_output_tensor_names=None):
+  """Exports inference graph for the model specified in the pipeline config.
+
+  Args:
+    input_type: Type of input for the graph. Can be one of [`image_tensor`,
+      `tf_example`].
+    pipeline_config: pipeline_pb2.TrainAndEvalPipelineConfig proto.
+    trained_checkpoint_prefix: Path to the trained checkpoint file.
+    output_directory: Path to write outputs.
+    input_shape: Sets a fixed shape for an `image_tensor` input. If not
+      specified, will default to [None, None, None, 3].
+    output_collection_name: Name of collection to add output tensors to.
+      If None, does not add output tensors to a collection.
+    additional_output_tensor_names: list of additional output
+      tensors to include in the frozen graph.
+  """
+  detection_model = model_builder.build(pipeline_config.model,
+                                        is_training=False)
+  _export_inference_graph(input_type, detection_model,
+                          pipeline_config.eval_config.use_moving_averages,
+                          trained_checkpoint_prefix,
+                          output_directory, additional_output_tensor_names,
+                          input_shape, output_collection_name,
+                          graph_hook_fn=None)
+  pipeline_config.eval_config.use_moving_averages = False
+  config_text = text_format.MessageToString(pipeline_config)
+  with tf.gfile.Open(
+      os.path.join(output_directory, 'pipeline.config'), 'wb') as f:
+    f.write(config_text)

research/mlperf_object_detection/Mask_RCNN/object_detection/exporter_test.py

+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Tests for object_detection.export_inference_graph."""
+import os
+import numpy as np
+import six
+import tensorflow as tf
+from google.protobuf import text_format
+from object_detection import exporter
+from object_detection.builders import model_builder
+from object_detection.core import model
+from object_detection.protos import pipeline_pb2
+
+if six.PY2:
+  import mock  # pylint: disable=g-import-not-at-top
+else:
+  from unittest import mock  # pylint: disable=g-import-not-at-top
+
+slim = tf.contrib.slim
+
+
+class FakeModel(model.DetectionModel):
+
+  def __init__(self, add_detection_keypoints=False, add_detection_masks=False):
+    self._add_detection_keypoints = add_detection_keypoints
+    self._add_detection_masks = add_detection_masks
+
+  def preprocess(self, inputs):
+    true_image_shapes = []  # Doesn't matter for the fake model.
+    return tf.identity(inputs), true_image_shapes
+
+  def predict(self, preprocessed_inputs, true_image_shapes):
+    return {'image': tf.layers.conv2d(preprocessed_inputs, 3, 1)}
+
+  def postprocess(self, prediction_dict, true_image_shapes):
+    with tf.control_dependencies(prediction_dict.values()):
+      postprocessed_tensors = {
+          'detection_boxes': tf.constant([[[0.0, 0.0, 0.5, 0.5],
+                                           [0.5, 0.5, 0.8, 0.8]],
+                                          [[0.5, 0.5, 1.0, 1.0],
+                                           [0.0, 0.0, 0.0, 0.0]]], tf.float32),
+          'detection_scores': tf.constant([[0.7, 0.6],
+                                           [0.9, 0.0]], tf.float32),
+          'detection_classes': tf.constant([[0, 1],
+                                            [1, 0]], tf.float32),
+          'num_detections': tf.constant([2, 1], tf.float32)
+      }
+      if self._add_detection_keypoints:
+        postprocessed_tensors['detection_keypoints'] = tf.constant(
+            np.arange(48).reshape([2, 2, 6, 2]), tf.float32)
+      if self._add_detection_masks:
+        postprocessed_tensors['detection_masks'] = tf.constant(
+            np.arange(64).reshape([2, 2, 4, 4]), tf.float32)
+    return postprocessed_tensors
+
+  def restore_map(self, checkpoint_path, fine_tune_checkpoint_type):
+    pass
+
+  def loss(self, prediction_dict, true_image_shapes):
+    pass
+
+
+class ExportInferenceGraphTest(tf.test.TestCase):
+
+  def _save_checkpoint_from_mock_model(self, checkpoint_path,
+                                       use_moving_averages):
+    g = tf.Graph()
+    with g.as_default():
+      mock_model = FakeModel()
+      preprocessed_inputs, true_image_shapes = mock_model.preprocess(
+          tf.placeholder(tf.float32, shape=[None, None, None, 3]))
+      predictions = mock_model.predict(preprocessed_inputs, true_image_shapes)
+      mock_model.postprocess(predictions, true_image_shapes)
+      if use_moving_averages:
+        tf.train.ExponentialMovingAverage(0.0).apply()
+      slim.get_or_create_global_step()
+      saver = tf.train.Saver()
+      init = tf.global_variables_initializer()
+      with self.test_session() as sess:
+        sess.run(init)
+        saver.save(sess, checkpoint_path)
+
+  def _load_inference_graph(self, inference_graph_path):
+    od_graph = tf.Graph()
+    with od_graph.as_default():
+      od_graph_def = tf.GraphDef()
+      with tf.gfile.GFile(inference_graph_path) as fid:
+        serialized_graph = fid.read()
+        od_graph_def.ParseFromString(serialized_graph)
+        tf.import_graph_def(od_graph_def, name='')
+    return od_graph
+
+  def _create_tf_example(self, image_array):
+    with self.test_session():
+      encoded_image = tf.image.encode_jpeg(tf.constant(image_array)).eval()
+    def _bytes_feature(value):
+      return tf.train.Feature(bytes_list=tf.train.BytesList(value=[value]))
+    example = tf.train.Example(features=tf.train.Features(feature={
+        'image/encoded': _bytes_feature(encoded_image),
+        'image/format': _bytes_feature('jpg'),
+        'image/source_id': _bytes_feature('image_id')
+    })).SerializeToString()
+    return example
+
+  def test_export_graph_with_image_tensor_input(self):
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    self._save_checkpoint_from_mock_model(trained_checkpoint_prefix,
+                                          use_moving_averages=False)
+    with mock.patch.object(
+        model_builder, 'build', autospec=True) as mock_builder:
+      mock_builder.return_value = FakeModel()
+      output_directory = os.path.join(tmp_dir, 'output')
+      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      pipeline_config.eval_config.use_moving_averages = False
+      exporter.export_inference_graph(
+          input_type='image_tensor',
+          pipeline_config=pipeline_config,
+          trained_checkpoint_prefix=trained_checkpoint_prefix,
+          output_directory=output_directory)
+      self.assertTrue(os.path.exists(os.path.join(
+          output_directory, 'saved_model', 'saved_model.pb')))
+
+  def test_export_graph_with_fixed_size_image_tensor_input(self):
+    input_shape = [1, 320, 320, 3]
+
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    self._save_checkpoint_from_mock_model(
+        trained_checkpoint_prefix, use_moving_averages=False)
+    with mock.patch.object(
+        model_builder, 'build', autospec=True) as mock_builder:
+      mock_builder.return_value = FakeModel()
+      output_directory = os.path.join(tmp_dir, 'output')
+      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      pipeline_config.eval_config.use_moving_averages = False
+      exporter.export_inference_graph(
+          input_type='image_tensor',
+          pipeline_config=pipeline_config,
+          trained_checkpoint_prefix=trained_checkpoint_prefix,
+          output_directory=output_directory,
+          input_shape=input_shape)
+      saved_model_path = os.path.join(output_directory, 'saved_model')
+      self.assertTrue(
+          os.path.exists(os.path.join(saved_model_path, 'saved_model.pb')))
+
+    with tf.Graph().as_default() as od_graph:
+      with self.test_session(graph=od_graph) as sess:
+        meta_graph = tf.saved_model.loader.load(
+            sess, [tf.saved_model.tag_constants.SERVING], saved_model_path)
+        signature = meta_graph.signature_def['serving_default']
+        input_tensor_name = signature.inputs['inputs'].name
+        image_tensor = od_graph.get_tensor_by_name(input_tensor_name)
+        self.assertSequenceEqual(image_tensor.get_shape().as_list(),
+                                 input_shape)
+
+  def test_export_graph_with_tf_example_input(self):
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    self._save_checkpoint_from_mock_model(trained_checkpoint_prefix,
+                                          use_moving_averages=False)
+    with mock.patch.object(
+        model_builder, 'build', autospec=True) as mock_builder:
+      mock_builder.return_value = FakeModel()
+      output_directory = os.path.join(tmp_dir, 'output')
+      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      pipeline_config.eval_config.use_moving_averages = False
+      exporter.export_inference_graph(
+          input_type='tf_example',
+          pipeline_config=pipeline_config,
+          trained_checkpoint_prefix=trained_checkpoint_prefix,
+          output_directory=output_directory)
+      self.assertTrue(os.path.exists(os.path.join(
+          output_directory, 'saved_model', 'saved_model.pb')))
+
+  def test_export_graph_with_encoded_image_string_input(self):
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    self._save_checkpoint_from_mock_model(trained_checkpoint_prefix,
+                                          use_moving_averages=False)
+    with mock.patch.object(
+        model_builder, 'build', autospec=True) as mock_builder:
+      mock_builder.return_value = FakeModel()
+      output_directory = os.path.join(tmp_dir, 'output')
+      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      pipeline_config.eval_config.use_moving_averages = False
+      exporter.export_inference_graph(
+          input_type='encoded_image_string_tensor',
+          pipeline_config=pipeline_config,
+          trained_checkpoint_prefix=trained_checkpoint_prefix,
+          output_directory=output_directory)
+      self.assertTrue(os.path.exists(os.path.join(
+          output_directory, 'saved_model', 'saved_model.pb')))
+
+  def _get_variables_in_checkpoint(self, checkpoint_file):
+    return set([
+        var_name
+        for var_name, _ in tf.train.list_variables(checkpoint_file)])
+
+  def test_replace_variable_values_with_moving_averages(self):
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    new_checkpoint_prefix = os.path.join(tmp_dir, 'new.ckpt')
+    self._save_checkpoint_from_mock_model(trained_checkpoint_prefix,
+                                          use_moving_averages=True)
+    graph = tf.Graph()
+    with graph.as_default():
+      fake_model = FakeModel()
+      preprocessed_inputs, true_image_shapes = fake_model.preprocess(
+          tf.placeholder(dtype=tf.float32, shape=[None, None, None, 3]))
+      predictions = fake_model.predict(preprocessed_inputs, true_image_shapes)
+      fake_model.postprocess(predictions, true_image_shapes)
+      exporter.replace_variable_values_with_moving_averages(
+          graph, trained_checkpoint_prefix, new_checkpoint_prefix)
+
+    expected_variables = set(['conv2d/bias', 'conv2d/kernel'])
+    variables_in_old_ckpt = self._get_variables_in_checkpoint(
+        trained_checkpoint_prefix)
+    self.assertIn('conv2d/bias/ExponentialMovingAverage',
+                  variables_in_old_ckpt)
+    self.assertIn('conv2d/kernel/ExponentialMovingAverage',
+                  variables_in_old_ckpt)
+    variables_in_new_ckpt = self._get_variables_in_checkpoint(
+        new_checkpoint_prefix)
+    self.assertTrue(expected_variables.issubset(variables_in_new_ckpt))
+    self.assertNotIn('conv2d/bias/ExponentialMovingAverage',
+                     variables_in_new_ckpt)
+    self.assertNotIn('conv2d/kernel/ExponentialMovingAverage',
+                     variables_in_new_ckpt)
+
+  def test_export_graph_with_moving_averages(self):
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    self._save_checkpoint_from_mock_model(trained_checkpoint_prefix,
+                                          use_moving_averages=True)
+    output_directory = os.path.join(tmp_dir, 'output')
+    with mock.patch.object(
+        model_builder, 'build', autospec=True) as mock_builder:
+      mock_builder.return_value = FakeModel()
+      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      pipeline_config.eval_config.use_moving_averages = True
+      exporter.export_inference_graph(
+          input_type='image_tensor',
+          pipeline_config=pipeline_config,
+          trained_checkpoint_prefix=trained_checkpoint_prefix,
+          output_directory=output_directory)
+      self.assertTrue(os.path.exists(os.path.join(
+          output_directory, 'saved_model', 'saved_model.pb')))
+    expected_variables = set(['conv2d/bias', 'conv2d/kernel', 'global_step'])
+    actual_variables = set(
+        [var_name for var_name, _ in tf.train.list_variables(output_directory)])
+    self.assertTrue(expected_variables.issubset(actual_variables))
+
+  def test_export_model_with_all_output_nodes(self):
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    self._save_checkpoint_from_mock_model(trained_checkpoint_prefix,
+                                          use_moving_averages=True)
+    output_directory = os.path.join(tmp_dir, 'output')
+    inference_graph_path = os.path.join(output_directory,
+                                        'frozen_inference_graph.pb')
+    with mock.patch.object(
+        model_builder, 'build', autospec=True) as mock_builder:
+      mock_builder.return_value = FakeModel(
+          add_detection_keypoints=True, add_detection_masks=True)
+      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      exporter.export_inference_graph(
+          input_type='image_tensor',
+          pipeline_config=pipeline_config,
+          trained_checkpoint_prefix=trained_checkpoint_prefix,
+          output_directory=output_directory)
+    inference_graph = self._load_inference_graph(inference_graph_path)
+    with self.test_session(graph=inference_graph):
+      inference_graph.get_tensor_by_name('image_tensor:0')
+      inference_graph.get_tensor_by_name('detection_boxes:0')
+      inference_graph.get_tensor_by_name('detection_scores:0')
+      inference_graph.get_tensor_by_name('detection_classes:0')
+      inference_graph.get_tensor_by_name('detection_keypoints:0')
+      inference_graph.get_tensor_by_name('detection_masks:0')
+      inference_graph.get_tensor_by_name('num_detections:0')
+
+  def test_export_model_with_detection_only_nodes(self):
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    self._save_checkpoint_from_mock_model(trained_checkpoint_prefix,
+                                          use_moving_averages=True)
+    output_directory = os.path.join(tmp_dir, 'output')
+    inference_graph_path = os.path.join(output_directory,
+                                        'frozen_inference_graph.pb')
+    with mock.patch.object(
+        model_builder, 'build', autospec=True) as mock_builder:
+      mock_builder.return_value = FakeModel(add_detection_masks=False)
+      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      exporter.export_inference_graph(
+          input_type='image_tensor',
+          pipeline_config=pipeline_config,
+          trained_checkpoint_prefix=trained_checkpoint_prefix,
+          output_directory=output_directory)
+    inference_graph = self._load_inference_graph(inference_graph_path)
+    with self.test_session(graph=inference_graph):
+      inference_graph.get_tensor_by_name('image_tensor:0')
+      inference_graph.get_tensor_by_name('detection_boxes:0')
+      inference_graph.get_tensor_by_name('detection_scores:0')
+      inference_graph.get_tensor_by_name('detection_classes:0')
+      inference_graph.get_tensor_by_name('num_detections:0')
+      with self.assertRaises(KeyError):
+        inference_graph.get_tensor_by_name('detection_keypoints:0')
+        inference_graph.get_tensor_by_name('detection_masks:0')
+
+  def test_export_and_run_inference_with_image_tensor(self):
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    self._save_checkpoint_from_mock_model(trained_checkpoint_prefix,
+                                          use_moving_averages=True)
+    output_directory = os.path.join(tmp_dir, 'output')
+    inference_graph_path = os.path.join(output_directory,
+                                        'frozen_inference_graph.pb')
+    with mock.patch.object(
+        model_builder, 'build', autospec=True) as mock_builder:
+      mock_builder.return_value = FakeModel(
+          add_detection_keypoints=True, add_detection_masks=True)
+      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      pipeline_config.eval_config.use_moving_averages = False
+      exporter.export_inference_graph(
+          input_type='image_tensor',
+          pipeline_config=pipeline_config,
+          trained_checkpoint_prefix=trained_checkpoint_prefix,
+          output_directory=output_directory)
+
+    inference_graph = self._load_inference_graph(inference_graph_path)
+    with self.test_session(graph=inference_graph) as sess:
+      image_tensor = inference_graph.get_tensor_by_name('image_tensor:0')
+      boxes = inference_graph.get_tensor_by_name('detection_boxes:0')
+      scores = inference_graph.get_tensor_by_name('detection_scores:0')
+      classes = inference_graph.get_tensor_by_name('detection_classes:0')
+      keypoints = inference_graph.get_tensor_by_name('detection_keypoints:0')
+      masks = inference_graph.get_tensor_by_name('detection_masks:0')
+      num_detections = inference_graph.get_tensor_by_name('num_detections:0')
+      (boxes_np, scores_np, classes_np, keypoints_np, masks_np,
+       num_detections_np) = sess.run(
+           [boxes, scores, classes, keypoints, masks, num_detections],
+           feed_dict={image_tensor: np.ones((2, 4, 4, 3)).astype(np.uint8)})
+      self.assertAllClose(boxes_np, [[[0.0, 0.0, 0.5, 0.5],
+                                      [0.5, 0.5, 0.8, 0.8]],
+                                     [[0.5, 0.5, 1.0, 1.0],
+                                      [0.0, 0.0, 0.0, 0.0]]])
+      self.assertAllClose(scores_np, [[0.7, 0.6],
+                                      [0.9, 0.0]])
+      self.assertAllClose(classes_np, [[1, 2],
+                                       [2, 1]])
+      self.assertAllClose(keypoints_np, np.arange(48).reshape([2, 2, 6, 2]))
+      self.assertAllClose(masks_np, np.arange(64).reshape([2, 2, 4, 4]))
+      self.assertAllClose(num_detections_np, [2, 1])
+
+  def _create_encoded_image_string(self, image_array_np, encoding_format):
+    od_graph = tf.Graph()
+    with od_graph.as_default():
+      if encoding_format == 'jpg':
+        encoded_string = tf.image.encode_jpeg(image_array_np)
+      elif encoding_format == 'png':
+        encoded_string = tf.image.encode_png(image_array_np)
+      else:
+        raise ValueError('Supports only the following formats: `jpg`, `png`')
+    with self.test_session(graph=od_graph):
+      return encoded_string.eval()
+
+  def test_export_and_run_inference_with_encoded_image_string_tensor(self):
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    self._save_checkpoint_from_mock_model(trained_checkpoint_prefix,
+                                          use_moving_averages=True)
+    output_directory = os.path.join(tmp_dir, 'output')
+    inference_graph_path = os.path.join(output_directory,
+                                        'frozen_inference_graph.pb')
+    with mock.patch.object(
+        model_builder, 'build', autospec=True) as mock_builder:
+      mock_builder.return_value = FakeModel(
+          add_detection_keypoints=True, add_detection_masks=True)
+      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      pipeline_config.eval_config.use_moving_averages = False
+      exporter.export_inference_graph(
+          input_type='encoded_image_string_tensor',
+          pipeline_config=pipeline_config,
+          trained_checkpoint_prefix=trained_checkpoint_prefix,
+          output_directory=output_directory)
+
+    inference_graph = self._load_inference_graph(inference_graph_path)
+    jpg_image_str = self._create_encoded_image_string(
+        np.ones((4, 4, 3)).astype(np.uint8), 'jpg')
+    png_image_str = self._create_encoded_image_string(
+        np.ones((4, 4, 3)).astype(np.uint8), 'png')
+    with self.test_session(graph=inference_graph) as sess:
+      image_str_tensor = inference_graph.get_tensor_by_name(
+          'encoded_image_string_tensor:0')
+      boxes = inference_graph.get_tensor_by_name('detection_boxes:0')
+      scores = inference_graph.get_tensor_by_name('detection_scores:0')
+      classes = inference_graph.get_tensor_by_name('detection_classes:0')
+      keypoints = inference_graph.get_tensor_by_name('detection_keypoints:0')
+      masks = inference_graph.get_tensor_by_name('detection_masks:0')
+      num_detections = inference_graph.get_tensor_by_name('num_detections:0')
+      for image_str in [jpg_image_str, png_image_str]:
+        image_str_batch_np = np.hstack([image_str]* 2)
+        (boxes_np, scores_np, classes_np, keypoints_np, masks_np,
+         num_detections_np) = sess.run(
+             [boxes, scores, classes, keypoints, masks, num_detections],
+             feed_dict={image_str_tensor: image_str_batch_np})
+        self.assertAllClose(boxes_np, [[[0.0, 0.0, 0.5, 0.5],
+                                        [0.5, 0.5, 0.8, 0.8]],
+                                       [[0.5, 0.5, 1.0, 1.0],
+                                        [0.0, 0.0, 0.0, 0.0]]])
+        self.assertAllClose(scores_np, [[0.7, 0.6],
+                                        [0.9, 0.0]])
+        self.assertAllClose(classes_np, [[1, 2],
+                                         [2, 1]])
+        self.assertAllClose(keypoints_np, np.arange(48).reshape([2, 2, 6, 2]))
+        self.assertAllClose(masks_np, np.arange(64).reshape([2, 2, 4, 4]))
+        self.assertAllClose(num_detections_np, [2, 1])
+
+  def test_raise_runtime_error_on_images_with_different_sizes(self):
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    self._save_checkpoint_from_mock_model(trained_checkpoint_prefix,
+                                          use_moving_averages=True)
+    output_directory = os.path.join(tmp_dir, 'output')
+    inference_graph_path = os.path.join(output_directory,
+                                        'frozen_inference_graph.pb')
+    with mock.patch.object(
+        model_builder, 'build', autospec=True) as mock_builder:
+      mock_builder.return_value = FakeModel(
+          add_detection_keypoints=True, add_detection_masks=True)
+      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      pipeline_config.eval_config.use_moving_averages = False
+      exporter.export_inference_graph(
+          input_type='encoded_image_string_tensor',
+          pipeline_config=pipeline_config,
+          trained_checkpoint_prefix=trained_checkpoint_prefix,
+          output_directory=output_directory)
+
+    inference_graph = self._load_inference_graph(inference_graph_path)
+    large_image = self._create_encoded_image_string(
+        np.ones((4, 4, 3)).astype(np.uint8), 'jpg')
+    small_image = self._create_encoded_image_string(
+        np.ones((2, 2, 3)).astype(np.uint8), 'jpg')
+
+    image_str_batch_np = np.hstack([large_image, small_image])
+    with self.test_session(graph=inference_graph) as sess:
+      image_str_tensor = inference_graph.get_tensor_by_name(
+          'encoded_image_string_tensor:0')
+      boxes = inference_graph.get_tensor_by_name('detection_boxes:0')
+      scores = inference_graph.get_tensor_by_name('detection_scores:0')
+      classes = inference_graph.get_tensor_by_name('detection_classes:0')
+      keypoints = inference_graph.get_tensor_by_name('detection_keypoints:0')
+      masks = inference_graph.get_tensor_by_name('detection_masks:0')
+      num_detections = inference_graph.get_tensor_by_name('num_detections:0')
+      with self.assertRaisesRegexp(tf.errors.InvalidArgumentError,
+                                   'TensorArray.*shape'):
+        sess.run(
+            [boxes, scores, classes, keypoints, masks, num_detections],
+            feed_dict={image_str_tensor: image_str_batch_np})
+
+  def test_export_and_run_inference_with_tf_example(self):
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    self._save_checkpoint_from_mock_model(trained_checkpoint_prefix,
+                                          use_moving_averages=True)
+    output_directory = os.path.join(tmp_dir, 'output')
+    inference_graph_path = os.path.join(output_directory,
+                                        'frozen_inference_graph.pb')
+    with mock.patch.object(
+        model_builder, 'build', autospec=True) as mock_builder:
+      mock_builder.return_value = FakeModel(
+          add_detection_keypoints=True, add_detection_masks=True)
+      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      pipeline_config.eval_config.use_moving_averages = False
+      exporter.export_inference_graph(
+          input_type='tf_example',
+          pipeline_config=pipeline_config,
+          trained_checkpoint_prefix=trained_checkpoint_prefix,
+          output_directory=output_directory)
+
+    inference_graph = self._load_inference_graph(inference_graph_path)
+    tf_example_np = np.expand_dims(self._create_tf_example(
+        np.ones((4, 4, 3)).astype(np.uint8)), axis=0)
+    with self.test_session(graph=inference_graph) as sess:
+      tf_example = inference_graph.get_tensor_by_name('tf_example:0')
+      boxes = inference_graph.get_tensor_by_name('detection_boxes:0')
+      scores = inference_graph.get_tensor_by_name('detection_scores:0')
+      classes = inference_graph.get_tensor_by_name('detection_classes:0')
+      keypoints = inference_graph.get_tensor_by_name('detection_keypoints:0')
+      masks = inference_graph.get_tensor_by_name('detection_masks:0')
+      num_detections = inference_graph.get_tensor_by_name('num_detections:0')
+      (boxes_np, scores_np, classes_np, keypoints_np, masks_np,
+       num_detections_np) = sess.run(
+           [boxes, scores, classes, keypoints, masks, num_detections],
+           feed_dict={tf_example: tf_example_np})
+      self.assertAllClose(boxes_np, [[[0.0, 0.0, 0.5, 0.5],
+                                      [0.5, 0.5, 0.8, 0.8]],
+                                     [[0.5, 0.5, 1.0, 1.0],
+                                      [0.0, 0.0, 0.0, 0.0]]])
+      self.assertAllClose(scores_np, [[0.7, 0.6],
+                                      [0.9, 0.0]])
+      self.assertAllClose(classes_np, [[1, 2],
+                                       [2, 1]])
+      self.assertAllClose(keypoints_np, np.arange(48).reshape([2, 2, 6, 2]))
+      self.assertAllClose(masks_np, np.arange(64).reshape([2, 2, 4, 4]))
+      self.assertAllClose(num_detections_np, [2, 1])
+
+  def test_write_frozen_graph(self):
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    self._save_checkpoint_from_mock_model(trained_checkpoint_prefix,
+                                          use_moving_averages=True)
+    output_directory = os.path.join(tmp_dir, 'output')
+    inference_graph_path = os.path.join(output_directory,
+                                        'frozen_inference_graph.pb')
+    tf.gfile.MakeDirs(output_directory)
+    with mock.patch.object(
+        model_builder, 'build', autospec=True) as mock_builder:
+      mock_builder.return_value = FakeModel(
+          add_detection_keypoints=True, add_detection_masks=True)
+      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      pipeline_config.eval_config.use_moving_averages = False
+      detection_model = model_builder.build(pipeline_config.model,
+                                            is_training=False)
+      outputs, _ = exporter._build_detection_graph(
+          input_type='tf_example',
+          detection_model=detection_model,
+          input_shape=None,
+          output_collection_name='inference_op',
+          graph_hook_fn=None)
+      output_node_names = ','.join(outputs.keys())
+      saver = tf.train.Saver()
+      input_saver_def = saver.as_saver_def()
+      frozen_graph_def = exporter.freeze_graph_with_def_protos(
+          input_graph_def=tf.get_default_graph().as_graph_def(),
+          input_saver_def=input_saver_def,
+          input_checkpoint=trained_checkpoint_prefix,
+          output_node_names=output_node_names,
+          restore_op_name='save/restore_all',
+          filename_tensor_name='save/Const:0',
+          clear_devices=True,
+          initializer_nodes='')
+      exporter.write_frozen_graph(inference_graph_path, frozen_graph_def)
+
+    inference_graph = self._load_inference_graph(inference_graph_path)
+    tf_example_np = np.expand_dims(self._create_tf_example(
+        np.ones((4, 4, 3)).astype(np.uint8)), axis=0)
+    with self.test_session(graph=inference_graph) as sess:
+      tf_example = inference_graph.get_tensor_by_name('tf_example:0')
+      boxes = inference_graph.get_tensor_by_name('detection_boxes:0')
+      scores = inference_graph.get_tensor_by_name('detection_scores:0')
+      classes = inference_graph.get_tensor_by_name('detection_classes:0')
+      keypoints = inference_graph.get_tensor_by_name('detection_keypoints:0')
+      masks = inference_graph.get_tensor_by_name('detection_masks:0')
+      num_detections = inference_graph.get_tensor_by_name('num_detections:0')
+      (boxes_np, scores_np, classes_np, keypoints_np, masks_np,
+       num_detections_np) = sess.run(
+           [boxes, scores, classes, keypoints, masks, num_detections],
+           feed_dict={tf_example: tf_example_np})
+      self.assertAllClose(boxes_np, [[[0.0, 0.0, 0.5, 0.5],
+                                      [0.5, 0.5, 0.8, 0.8]],
+                                     [[0.5, 0.5, 1.0, 1.0],
+                                      [0.0, 0.0, 0.0, 0.0]]])
+      self.assertAllClose(scores_np, [[0.7, 0.6],
+                                      [0.9, 0.0]])
+      self.assertAllClose(classes_np, [[1, 2],
+                                       [2, 1]])
+      self.assertAllClose(keypoints_np, np.arange(48).reshape([2, 2, 6, 2]))
+      self.assertAllClose(masks_np, np.arange(64).reshape([2, 2, 4, 4]))
+      self.assertAllClose(num_detections_np, [2, 1])
+
+  def test_export_graph_saves_pipeline_file(self):
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    self._save_checkpoint_from_mock_model(trained_checkpoint_prefix,
+                                          use_moving_averages=True)
+    output_directory = os.path.join(tmp_dir, 'output')
+    with mock.patch.object(
+        model_builder, 'build', autospec=True) as mock_builder:
+      mock_builder.return_value = FakeModel()
+      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      exporter.export_inference_graph(
+          input_type='image_tensor',
+          pipeline_config=pipeline_config,
+          trained_checkpoint_prefix=trained_checkpoint_prefix,
+          output_directory=output_directory)
+      expected_pipeline_path = os.path.join(
+          output_directory, 'pipeline.config')
+      self.assertTrue(os.path.exists(expected_pipeline_path))
+
+      written_pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      with tf.gfile.GFile(expected_pipeline_path, 'r') as f:
+        proto_str = f.read()
+        text_format.Merge(proto_str, written_pipeline_config)
+        self.assertProtoEquals(pipeline_config, written_pipeline_config)
+
+  def test_export_saved_model_and_run_inference(self):
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    self._save_checkpoint_from_mock_model(trained_checkpoint_prefix,
+                                          use_moving_averages=False)
+    output_directory = os.path.join(tmp_dir, 'output')
+    saved_model_path = os.path.join(output_directory, 'saved_model')
+
+    with mock.patch.object(
+        model_builder, 'build', autospec=True) as mock_builder:
+      mock_builder.return_value = FakeModel(
+          add_detection_keypoints=True, add_detection_masks=True)
+      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      pipeline_config.eval_config.use_moving_averages = False
+      exporter.export_inference_graph(
+          input_type='tf_example',
+          pipeline_config=pipeline_config,
+          trained_checkpoint_prefix=trained_checkpoint_prefix,
+          output_directory=output_directory)
+
+    tf_example_np = np.hstack([self._create_tf_example(
+        np.ones((4, 4, 3)).astype(np.uint8))] * 2)
+    with tf.Graph().as_default() as od_graph:
+      with self.test_session(graph=od_graph) as sess:
+        meta_graph = tf.saved_model.loader.load(
+            sess, [tf.saved_model.tag_constants.SERVING], saved_model_path)
+
+        signature = meta_graph.signature_def['serving_default']
+        input_tensor_name = signature.inputs['inputs'].name
+        tf_example = od_graph.get_tensor_by_name(input_tensor_name)
+
+        boxes = od_graph.get_tensor_by_name(
+            signature.outputs['detection_boxes'].name)
+        scores = od_graph.get_tensor_by_name(
+            signature.outputs['detection_scores'].name)
+        classes = od_graph.get_tensor_by_name(
+            signature.outputs['detection_classes'].name)
+        keypoints = od_graph.get_tensor_by_name(
+            signature.outputs['detection_keypoints'].name)
+        masks = od_graph.get_tensor_by_name(
+            signature.outputs['detection_masks'].name)
+        num_detections = od_graph.get_tensor_by_name(
+            signature.outputs['num_detections'].name)
+
+        (boxes_np, scores_np, classes_np, keypoints_np, masks_np,
+         num_detections_np) = sess.run(
+             [boxes, scores, classes, keypoints, masks, num_detections],
+             feed_dict={tf_example: tf_example_np})
+        self.assertAllClose(boxes_np, [[[0.0, 0.0, 0.5, 0.5],
+                                        [0.5, 0.5, 0.8, 0.8]],
+                                       [[0.5, 0.5, 1.0, 1.0],
+                                        [0.0, 0.0, 0.0, 0.0]]])
+        self.assertAllClose(scores_np, [[0.7, 0.6],
+                                        [0.9, 0.0]])
+        self.assertAllClose(classes_np, [[1, 2],
+                                         [2, 1]])
+        self.assertAllClose(keypoints_np, np.arange(48).reshape([2, 2, 6, 2]))
+        self.assertAllClose(masks_np, np.arange(64).reshape([2, 2, 4, 4]))
+        self.assertAllClose(num_detections_np, [2, 1])
+
+  def test_write_saved_model(self):
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    self._save_checkpoint_from_mock_model(trained_checkpoint_prefix,
+                                          use_moving_averages=False)
+    output_directory = os.path.join(tmp_dir, 'output')
+    saved_model_path = os.path.join(output_directory, 'saved_model')
+    tf.gfile.MakeDirs(output_directory)
+    with mock.patch.object(
+        model_builder, 'build', autospec=True) as mock_builder:
+      mock_builder.return_value = FakeModel(
+          add_detection_keypoints=True, add_detection_masks=True)
+      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      pipeline_config.eval_config.use_moving_averages = False
+      detection_model = model_builder.build(pipeline_config.model,
+                                            is_training=False)
+      outputs, placeholder_tensor = exporter._build_detection_graph(
+          input_type='tf_example',
+          detection_model=detection_model,
+          input_shape=None,
+          output_collection_name='inference_op',
+          graph_hook_fn=None)
+      output_node_names = ','.join(outputs.keys())
+      saver = tf.train.Saver()
+      input_saver_def = saver.as_saver_def()
+      frozen_graph_def = exporter.freeze_graph_with_def_protos(
+          input_graph_def=tf.get_default_graph().as_graph_def(),
+          input_saver_def=input_saver_def,
+          input_checkpoint=trained_checkpoint_prefix,
+          output_node_names=output_node_names,
+          restore_op_name='save/restore_all',
+          filename_tensor_name='save/Const:0',
+          clear_devices=True,
+          initializer_nodes='')
+      exporter.write_saved_model(
+          saved_model_path=saved_model_path,
+          frozen_graph_def=frozen_graph_def,
+          inputs=placeholder_tensor,
+          outputs=outputs)
+
+    tf_example_np = np.hstack([self._create_tf_example(
+        np.ones((4, 4, 3)).astype(np.uint8))] * 2)
+    with tf.Graph().as_default() as od_graph:
+      with self.test_session(graph=od_graph) as sess:
+        meta_graph = tf.saved_model.loader.load(
+            sess, [tf.saved_model.tag_constants.SERVING], saved_model_path)
+
+        signature = meta_graph.signature_def['serving_default']
+        input_tensor_name = signature.inputs['inputs'].name
+        tf_example = od_graph.get_tensor_by_name(input_tensor_name)
+
+        boxes = od_graph.get_tensor_by_name(
+            signature.outputs['detection_boxes'].name)
+        scores = od_graph.get_tensor_by_name(
+            signature.outputs['detection_scores'].name)
+        classes = od_graph.get_tensor_by_name(
+            signature.outputs['detection_classes'].name)
+        keypoints = od_graph.get_tensor_by_name(
+            signature.outputs['detection_keypoints'].name)
+        masks = od_graph.get_tensor_by_name(
+            signature.outputs['detection_masks'].name)
+        num_detections = od_graph.get_tensor_by_name(
+            signature.outputs['num_detections'].name)
+
+        (boxes_np, scores_np, classes_np, keypoints_np, masks_np,
+         num_detections_np) = sess.run(
+             [boxes, scores, classes, keypoints, masks, num_detections],
+             feed_dict={tf_example: tf_example_np})
+        self.assertAllClose(boxes_np, [[[0.0, 0.0, 0.5, 0.5],
+                                        [0.5, 0.5, 0.8, 0.8]],
+                                       [[0.5, 0.5, 1.0, 1.0],
+                                        [0.0, 0.0, 0.0, 0.0]]])
+        self.assertAllClose(scores_np, [[0.7, 0.6],
+                                        [0.9, 0.0]])
+        self.assertAllClose(classes_np, [[1, 2],
+                                         [2, 1]])
+        self.assertAllClose(keypoints_np, np.arange(48).reshape([2, 2, 6, 2]))
+        self.assertAllClose(masks_np, np.arange(64).reshape([2, 2, 4, 4]))
+        self.assertAllClose(num_detections_np, [2, 1])
+
+  def test_export_checkpoint_and_run_inference(self):
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    self._save_checkpoint_from_mock_model(trained_checkpoint_prefix,
+                                          use_moving_averages=False)
+    output_directory = os.path.join(tmp_dir, 'output')
+    model_path = os.path.join(output_directory, 'model.ckpt')
+    meta_graph_path = model_path + '.meta'
+
+    with mock.patch.object(
+        model_builder, 'build', autospec=True) as mock_builder:
+      mock_builder.return_value = FakeModel(
+          add_detection_keypoints=True, add_detection_masks=True)
+      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      pipeline_config.eval_config.use_moving_averages = False
+      exporter.export_inference_graph(
+          input_type='tf_example',
+          pipeline_config=pipeline_config,
+          trained_checkpoint_prefix=trained_checkpoint_prefix,
+          output_directory=output_directory)
+
+    tf_example_np = np.hstack([self._create_tf_example(
+        np.ones((4, 4, 3)).astype(np.uint8))] * 2)
+    with tf.Graph().as_default() as od_graph:
+      with self.test_session(graph=od_graph) as sess:
+        new_saver = tf.train.import_meta_graph(meta_graph_path)
+        new_saver.restore(sess, model_path)
+
+        tf_example = od_graph.get_tensor_by_name('tf_example:0')
+        boxes = od_graph.get_tensor_by_name('detection_boxes:0')
+        scores = od_graph.get_tensor_by_name('detection_scores:0')
+        classes = od_graph.get_tensor_by_name('detection_classes:0')
+        keypoints = od_graph.get_tensor_by_name('detection_keypoints:0')
+        masks = od_graph.get_tensor_by_name('detection_masks:0')
+        num_detections = od_graph.get_tensor_by_name('num_detections:0')
+        (boxes_np, scores_np, classes_np, keypoints_np, masks_np,
+         num_detections_np) = sess.run(
+             [boxes, scores, classes, keypoints, masks, num_detections],
+             feed_dict={tf_example: tf_example_np})
+        self.assertAllClose(boxes_np, [[[0.0, 0.0, 0.5, 0.5],
+                                        [0.5, 0.5, 0.8, 0.8]],
+                                       [[0.5, 0.5, 1.0, 1.0],
+                                        [0.0, 0.0, 0.0, 0.0]]])
+        self.assertAllClose(scores_np, [[0.7, 0.6],
+                                        [0.9, 0.0]])
+        self.assertAllClose(classes_np, [[1, 2],
+                                         [2, 1]])
+        self.assertAllClose(keypoints_np, np.arange(48).reshape([2, 2, 6, 2]))
+        self.assertAllClose(masks_np, np.arange(64).reshape([2, 2, 4, 4]))
+        self.assertAllClose(num_detections_np, [2, 1])
+
+  def test_write_graph_and_checkpoint(self):
+    tmp_dir = self.get_temp_dir()
+    trained_checkpoint_prefix = os.path.join(tmp_dir, 'model.ckpt')
+    self._save_checkpoint_from_mock_model(trained_checkpoint_prefix,
+                                          use_moving_averages=False)
+    output_directory = os.path.join(tmp_dir, 'output')
+    model_path = os.path.join(output_directory, 'model.ckpt')
+    meta_graph_path = model_path + '.meta'
+    tf.gfile.MakeDirs(output_directory)
+    with mock.patch.object(
+        model_builder, 'build', autospec=True) as mock_builder:
+      mock_builder.return_value = FakeModel(
+          add_detection_keypoints=True, add_detection_masks=True)
+      pipeline_config = pipeline_pb2.TrainEvalPipelineConfig()
+      pipeline_config.eval_config.use_moving_averages = False
+      detection_model = model_builder.build(pipeline_config.model,
+                                            is_training=False)
+      exporter._build_detection_graph(
+          input_type='tf_example',
+          detection_model=detection_model,
+          input_shape=None,
+          output_collection_name='inference_op',
+          graph_hook_fn=None)
+      saver = tf.train.Saver()
+      input_saver_def = saver.as_saver_def()
+      exporter.write_graph_and_checkpoint(
+          inference_graph_def=tf.get_default_graph().as_graph_def(),
+          model_path=model_path,
+          input_saver_def=input_saver_def,
+          trained_checkpoint_prefix=trained_checkpoint_prefix)
+
+    tf_example_np = np.hstack([self._create_tf_example(
+        np.ones((4, 4, 3)).astype(np.uint8))] * 2)
+    with tf.Graph().as_default() as od_graph:
+      with self.test_session(graph=od_graph) as sess:
+        new_saver = tf.train.import_meta_graph(meta_graph_path)
+        new_saver.restore(sess, model_path)
+
+        tf_example = od_graph.get_tensor_by_name('tf_example:0')
+        boxes = od_graph.get_tensor_by_name('detection_boxes:0')
+        scores = od_graph.get_tensor_by_name('detection_scores:0')
+        classes = od_graph.get_tensor_by_name('detection_classes:0')
+        keypoints = od_graph.get_tensor_by_name('detection_keypoints:0')
+        masks = od_graph.get_tensor_by_name('detection_masks:0')
+        num_detections = od_graph.get_tensor_by_name('num_detections:0')
+        (boxes_np, scores_np, classes_np, keypoints_np, masks_np,
+         num_detections_np) = sess.run(
+             [boxes, scores, classes, keypoints, masks, num_detections],
+             feed_dict={tf_example: tf_example_np})
+        self.assertAllClose(boxes_np, [[[0.0, 0.0, 0.5, 0.5],
+                                        [0.5, 0.5, 0.8, 0.8]],
+                                       [[0.5, 0.5, 1.0, 1.0],
+                                        [0.0, 0.0, 0.0, 0.0]]])
+        self.assertAllClose(scores_np, [[0.7, 0.6],
+                                        [0.9, 0.0]])
+        self.assertAllClose(classes_np, [[1, 2],
+                                         [2, 1]])
+        self.assertAllClose(keypoints_np, np.arange(48).reshape([2, 2, 6, 2]))
+        self.assertAllClose(masks_np, np.arange(64).reshape([2, 2, 4, 4]))
+        self.assertAllClose(num_detections_np, [2, 1])
+
+
+if __name__ == '__main__':
+  tf.test.main()

research/mlperf_object_detection/Mask_RCNN/object_detection/g3doc/configuring_jobs.md

+# Configuring the Object Detection Training Pipeline
+
+## Overview
+
+The Tensorflow Object Detection API uses protobuf files to configure the
+training and evaluation process. The schema for the training pipeline can be
+found in object_detection/protos/pipeline.proto. At a high level, the config
+file is split into 5 parts:
+
+1. The `model` configuration. This defines what type of model will be trained
+(ie. meta-architecture, feature extractor).
+2. The `train_config`, which decides what parameters should be used to train
+model parameters (ie. SGD parameters, input preprocessing and feature extractor
+initialization values).
+3. The `eval_config`, which determines what set of metrics will be reported for
+evaluation (currently we only support the PASCAL VOC metrics).
+4. The `train_input_config`, which defines what dataset the model should be
+trained on.
+5. The `eval_input_config`, which defines what dataset the model will be
+evaluated on. Typically this should be different than the training input
+dataset.
+
+A skeleton configuration file is shown below:
+
+```
+model {
+(... Add model config here...)
+}
+
+train_config : {
+(... Add train_config here...)
+}
+
+train_input_reader: {
+(... Add train_input configuration here...)
+}
+
+eval_config: {
+}
+
+eval_input_reader: {
+(... Add eval_input configuration here...)
+}
+```
+
+## Picking Model Parameters
+
+There are a large number of model parameters to configure. The best settings
+will depend on your given application. Faster R-CNN models are better suited to
+cases where high accuracy is desired and latency is of lower priority.
+Conversely, if processing time is the most important factor, SSD models are
+recommended. Read [our paper](https://arxiv.org/abs/1611.10012) for a more
+detailed discussion on the speed vs accuracy tradeoff.
+
+To help new users get started, sample model configurations have been provided
+in the object_detection/samples/configs folder. The contents of these
+configuration files can be pasted into `model` field of the skeleton
+configuration. Users should note that the `num_classes` field should be changed
+to a value suited for the dataset the user is training on.
+
+## Defining Inputs
+
+The Tensorflow Object Detection API accepts inputs in the TFRecord file format.
+Users must specify the locations of both the training and evaluation files.
+Additionally, users should also specify a label map, which define the mapping
+between a class id and class name. The label map should be identical between
+training and evaluation datasets.
+
+An example input configuration looks as follows:
+
+```
+tf_record_input_reader {
+  input_path: "/usr/home/username/data/train.record"
+}
+label_map_path: "/usr/home/username/data/label_map.pbtxt"
+```
+
+Users should substitute the `input_path` and `label_map_path` arguments and
+insert the input configuration into the `train_input_reader` and
+`eval_input_reader` fields in the skeleton configuration. Note that the paths
+can also point to Google Cloud Storage buckets (ie.
+"gs://project_bucket/train.record") for use on Google Cloud.
+
+## Configuring the Trainer
+
+The `train_config` defines parts of the training process:
+
+1. Model parameter initialization.
+2. Input preprocessing.
+3. SGD parameters.
+
+A sample `train_config` is below:
+
+```
+batch_size: 1
+optimizer {
+  momentum_optimizer: {
+    learning_rate: {
+      manual_step_learning_rate {
+        initial_learning_rate: 0.0002
+        schedule {
+          step: 0
+          learning_rate: .0002
+        }
+        schedule {
+          step: 900000
+          learning_rate: .00002
+        }
+        schedule {
+          step: 1200000
+          learning_rate: .000002
+        }
+      }
+    }
+    momentum_optimizer_value: 0.9
+  }
+  use_moving_average: false
+}
+fine_tune_checkpoint: "/usr/home/username/tmp/model.ckpt-#####"
+from_detection_checkpoint: true
+gradient_clipping_by_norm: 10.0
+data_augmentation_options {
+  random_horizontal_flip {
+  }
+}
+```
+
+### Model Parameter Initialization
+
+While optional, it is highly recommended that users utilize other object
+detection checkpoints. Training an object detector from scratch can take days.
+To speed up the training process, it is recommended that users re-use the
+feature extractor parameters from a pre-existing object classification or
+detection checkpoint. `train_config` provides two fields to specify
+pre-existing checkpoints: `fine_tune_checkpoint` and
+`from_detection_checkpoint`. `fine_tune_checkpoint` should provide a path to
+the pre-existing checkpoint
+(ie:"/usr/home/username/checkpoint/model.ckpt-#####").
+`from_detection_checkpoint` is a boolean value. If false, it assumes the
+checkpoint was from an object classification checkpoint. Note that starting
+from a detection checkpoint will usually result in a faster training job than
+a classification checkpoint.
+
+The list of provided checkpoints can be found [here](detection_model_zoo.md).
+
+### Input Preprocessing
+
+The `data_augmentation_options` in `train_config` can be used to specify
+how training data can be modified. This field is optional.
+
+### SGD Parameters
+
+The remainings parameters in `train_config` are hyperparameters for gradient
+descent. Please note that the optimal learning rates provided in these
+configuration files may depend on the specifics of the training setup (e.g.
+number of workers, gpu type).
+
+## Configuring the Evaluator
+
+Currently evaluation is fixed to generating metrics as defined by the PASCAL VOC
+challenge. The parameters for `eval_config` are set to reasonable defaults and
+typically do not need to be configured.

research/mlperf_object_detection/Mask_RCNN/object_detection/g3doc/defining_your_own_model.md

+# So you want to create a new model!
+
+In this section, we discuss some of the abstractions that we use
+for defining detection models. If you would like to define a new model
+architecture for detection and use it in the Tensorflow Detection API,
+then this section should also serve as a high level guide to the files that you
+will need to edit to get your new model working.
+
+## DetectionModels (`object_detection/core/model.py`)
+
+In order to be trained, evaluated, and exported for serving  using our
+provided binaries, all models under the Tensorflow Object Detection API must
+implement the `DetectionModel` interface (see the full definition in `object_detection/core/model.py`).  In particular,
+each of these models are responsible for implementing 5 functions:
+
+* `preprocess`: Run any preprocessing (e.g., scaling/shifting/reshaping) of
+  input values that is necessary prior to running the detector on an input
+  image.
+* `predict`: Produce “raw” prediction tensors that can be passed to loss or
+  postprocess functions.
+* `postprocess`: Convert predicted output tensors to final detections.
+* `loss`: Compute scalar loss tensors with respect to provided groundtruth.
+* `restore`: Load a checkpoint into the Tensorflow graph.
+
+Given a `DetectionModel` at training time, we pass each image batch through
+the following sequence of functions to compute a loss which can be optimized via
+SGD:
+
+```
+inputs (images tensor) -> preprocess -> predict -> loss -> outputs (loss tensor)
+```
+
+And at eval time, we pass each image batch through the following sequence of
+functions to produce a set of detections:
+
+```
+inputs (images tensor) -> preprocess -> predict -> postprocess ->
+  outputs (boxes tensor, scores tensor, classes tensor, num_detections tensor)
+```
+
+Some conventions to be aware of:
+
+* `DetectionModel`s should make no assumptions about the input size or aspect
+  ratio --- they are responsible for doing any resize/reshaping necessary
+  (see docstring for the `preprocess` function).
+* Output classes are always integers in the range `[0, num_classes)`.
+  Any mapping of these integers to semantic labels is to be handled outside
+  of this class.  We never explicitly emit a “background class” --- thus 0 is
+  the first non-background class and any logic of predicting and removing
+  implicit background classes must be handled internally by the implementation.
+* Detected boxes are to be interpreted as being in
+  `[y_min, x_min, y_max, x_max]` format and normalized relative to the
+  image window.
+* We do not specifically assume any kind of probabilistic interpretation of the
+  scores --- the only important thing is their relative ordering. Thus
+  implementations of the postprocess function are free to output logits,
+  probabilities, calibrated probabilities, or anything else.
+
+## Defining a new Faster R-CNN or SSD Feature Extractor
+
+In most cases, you probably will not implement a `DetectionModel` from scratch
+--- instead you might create a new feature extractor to be used by one of the
+SSD or Faster R-CNN meta-architectures.  (We think of meta-architectures as
+classes that define entire families of models using the `DetectionModel`
+abstraction).
+
+Note: For the following discussion to make sense, we recommend first becoming
+familiar with the [Faster R-CNN](https://arxiv.org/abs/1506.01497) paper.
+
+Let’s now imagine that you have invented a brand new network architecture
+(say, “InceptionV100”) for classification and want to see how InceptionV100
+would behave as a feature extractor for detection (say, with Faster R-CNN).
+A similar procedure would hold for SSD models, but we’ll discuss Faster R-CNN.
+
+To use InceptionV100, we will have to define a new
+`FasterRCNNFeatureExtractor` and pass it to our `FasterRCNNMetaArch`
+constructor as input.  See
+`object_detection/meta_architectures/faster_rcnn_meta_arch.py` for definitions
+of `FasterRCNNFeatureExtractor` and `FasterRCNNMetaArch`, respectively.
+A `FasterRCNNFeatureExtractor` must define a few
+functions:
+
+* `preprocess`: Run any preprocessing of input values that is necessary prior
+  to running the detector on an input image.
+* `_extract_proposal_features`: Extract first stage Region Proposal Network
+  (RPN) features.
+* `_extract_box_classifier_features`: Extract second stage Box Classifier
+  features.
+* `restore_from_classification_checkpoint_fn`: Load a checkpoint into the
+  Tensorflow graph.
+
+See the `object_detection/models/faster_rcnn_resnet_v1_feature_extractor.py`
+definition as one example. Some remarks:
+
+* We typically initialize the weights of this feature extractor
+  using those from the
+  [Slim Resnet-101 classification checkpoint](https://github.com/tensorflow/models/tree/master/research/slim#pre-trained-models),
+  and we know
+  that images were preprocessed when training this checkpoint
+  by subtracting a channel mean from each input
+  image.  Thus, we implement the preprocess function to replicate the same
+  channel mean subtraction behavior.
+* The “full” resnet classification network defined in slim is cut into two
+  parts --- all but the last “resnet block” is put into the
+  `_extract_proposal_features` function and the final block is separately
+  defined in the `_extract_box_classifier_features function`.  In general,
+  some experimentation may be required to decide on an optimal layer at
+  which to “cut” your feature extractor into these two pieces for Faster R-CNN.
+
+## Register your model for configuration
+
+Assuming that your new feature extractor does not require nonstandard
+configuration, you will want to ideally be able to simply change the
+“feature_extractor.type” fields in your configuration protos to point to a
+new feature extractor.  In order for our API to know how to understand this
+new type though, you will first have to register your new feature
+extractor with the model builder (`object_detection/builders/model_builder.py`),
+whose job is to create models from config protos..
+
+Registration is simple --- just add a pointer to the new Feature Extractor
+class that you have defined in one of the SSD or Faster R-CNN Feature
+Extractor Class maps at the top of the
+`object_detection/builders/model_builder.py` file.
+We recommend adding a test in `object_detection/builders/model_builder_test.py`
+to make sure that parsing your proto will work as expected.
+
+## Taking your new model for a spin
+
+After registration you are ready to go with your model!  Some final tips:
+
+* To save time debugging, try running your configuration file locally first
+  (both training and evaluation).
+* Do a sweep of learning rates to figure out which learning rate is best
+  for your model.
+* A small but often important detail: you may find it necessary to disable
+  batchnorm training (that is, load the batch norm parameters from the
+  classification checkpoint, but do not update them during gradient descent).

research/mlperf_object_detection/Mask_RCNN/object_detection/g3doc/detection_model_zoo.md

+# Tensorflow detection model zoo
+
+We provide a collection of detection models pre-trained on the [COCO
+dataset](http://mscoco.org), the [Kitti dataset](http://www.cvlibs.net/datasets/kitti/), and the
+[Open Images dataset](https://github.com/openimages/dataset). These models can
+be useful for
+out-of-the-box inference if you are interested in categories already in COCO
+(e.g., humans, cars, etc) or in Open Images (e.g.,
+surfboard, jacuzzi, etc). They are also useful for initializing your models when
+training on novel datasets.
+
+In the table below, we list each such pre-trained model including:
+
+* a model name that corresponds to a config file that was used to train this
+  model in the `samples/configs` directory,
+* a download link to a tar.gz file containing the pre-trained model,
+* model speed --- we report running time in ms per 600x600 image (including all
+  pre and post-processing), but please be
+  aware that these timings depend highly on one's specific hardware
+  configuration (these timings were performed using an Nvidia
+  GeForce GTX TITAN X card) and should be treated more as relative timings in
+  many cases. Also note that desktop GPU timing does not always reflect mobile
+  run time. For example Mobilenet V2 is faster on mobile devices than Mobilenet
+  V1, but is slightly slower on desktop GPU.
+* detector performance on subset of the COCO validation set or Open Images test split as measured by the dataset-specific mAP measure.
+  Here, higher is better, and we only report bounding box mAP rounded to the
+  nearest integer.
+* Output types (`Boxes`, and `Masks` if applicable )
+
+You can un-tar each tar.gz file via, e.g.,:
+
+```
+tar -xzvf ssd_mobilenet_v1_coco.tar.gz
+```
+
+Inside the un-tar'ed directory, you will find:
+
+* a graph proto (`graph.pbtxt`)
+* a checkpoint
+  (`model.ckpt.data-00000-of-00001`, `model.ckpt.index`, `model.ckpt.meta`)
+* a frozen graph proto with weights baked into the graph as constants
+  (`frozen_inference_graph.pb`) to be used for out of the box inference
+    (try this out in the Jupyter notebook!)
+* a config file (`pipeline.config`) which was used to generate the graph.  These
+  directly correspond to a config file in the
+  [samples/configs](https://github.com/tensorflow/models/tree/master/research/object_detection/samples/configs)) directory but often with a modified score threshold.  In the case
+  of the heavier Faster R-CNN models, we also provide a version of the model
+  that uses a highly reduced number of proposals for speed.
+
+Some remarks on frozen inference graphs:
+
+* If you try to evaluate the frozen graph, you may find performance numbers for
+  some of the models to be slightly lower than what we report in the below
+  tables.  This is because we discard detections with scores below a
+  threshold (typically 0.3) when creating the frozen graph.  This corresponds
+  effectively to picking a point on the precision recall curve of
+  a detector (and discarding the part past that point), which negatively impacts
+  standard mAP metrics.
+* Our frozen inference graphs are generated using the
+  [v1.5.0](https://github.com/tensorflow/tensorflow/tree/v1.5.0)
+  release version of Tensorflow and we do not guarantee that these will work
+  with other versions; this being said, each frozen inference graph can be
+  regenerated using your current version of Tensorflow by re-running the
+  [exporter](https://github.com/tensorflow/models/blob/master/research/object_detection/g3doc/exporting_models.md),
+  pointing it at the model directory as well as the config file inside of it.
+
+
+## COCO-trained models {#coco-models}
+
+| Model name  | Speed (ms) | COCO mAP[^1] | Outputs |
+| ------------ | :--------------: | :--------------: | :-------------: |
+| [ssd_mobilenet_v1_coco](http://download.tensorflow.org/models/object_detection/ssd_mobilenet_v1_coco_2017_11_17.tar.gz) | 30 | 21 | Boxes |
+| [ssd_mobilenet_v2_coco](http://download.tensorflow.org/models/object_detection/ssd_mobilenet_v2_coco_2018_03_29.tar.gz) | 31 | 22 | Boxes |
+| [ssdlite_mobilenet_v2_coco](http://download.tensorflow.org/models/object_detection/ssdlite_mobilenet_v2_coco_2018_05_09.tar.gz) | 27 | 22 | Boxes |
+| [ssd_inception_v2_coco](http://download.tensorflow.org/models/object_detection/ssd_inception_v2_coco_2017_11_17.tar.gz) | 42 | 24 | Boxes |
+| [faster_rcnn_inception_v2_coco](http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_v2_coco_2018_01_28.tar.gz) | 58 | 28 | Boxes |
+| [faster_rcnn_resnet50_coco](http://download.tensorflow.org/models/object_detection/faster_rcnn_resnet50_coco_2018_01_28.tar.gz) | 89 | 30 | Boxes |
+| [faster_rcnn_resnet50_lowproposals_coco](http://download.tensorflow.org/models/object_detection/faster_rcnn_resnet50_lowproposals_coco_2018_01_28.tar.gz) | 64 |  | Boxes |
+| [rfcn_resnet101_coco](http://download.tensorflow.org/models/object_detection/rfcn_resnet101_coco_2018_01_28.tar.gz)  | 92 | 30 | Boxes |
+| [faster_rcnn_resnet101_coco](http://download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_coco_2018_01_28.tar.gz) | 106 | 32 | Boxes |
+| [faster_rcnn_resnet101_lowproposals_coco](http://download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_lowproposals_coco_2018_01_28.tar.gz) | 82 |  | Boxes |
+| [faster_rcnn_inception_resnet_v2_atrous_coco](http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz) | 620 | 37 | Boxes |
+| [faster_rcnn_inception_resnet_v2_atrous_lowproposals_coco](http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_lowproposals_coco_2018_01_28.tar.gz) | 241 |  | Boxes |
+| [faster_rcnn_nas](http://download.tensorflow.org/models/object_detection/faster_rcnn_nas_coco_2018_01_28.tar.gz) | 1833 | 43 | Boxes |
+| [faster_rcnn_nas_lowproposals_coco](http://download.tensorflow.org/models/object_detection/faster_rcnn_nas_lowproposals_coco_2018_01_28.tar.gz) | 540 |  | Boxes |
+| [mask_rcnn_inception_resnet_v2_atrous_coco](http://download.tensorflow.org/models/object_detection/mask_rcnn_inception_resnet_v2_atrous_coco_2018_01_28.tar.gz) | 771 | 36 | Masks |
+| [mask_rcnn_inception_v2_coco](http://download.tensorflow.org/models/object_detection/mask_rcnn_inception_v2_coco_2018_01_28.tar.gz) | 79 | 25 | Masks |
+| [mask_rcnn_resnet101_atrous_coco](http://download.tensorflow.org/models/object_detection/mask_rcnn_resnet101_atrous_coco_2018_01_28.tar.gz) | 470 | 33 | Masks |
+| [mask_rcnn_resnet50_atrous_coco](http://download.tensorflow.org/models/object_detection/mask_rcnn_resnet50_atrous_coco_2018_01_28.tar.gz) | 343 | 29 | Masks |
+
+
+
+## Kitti-trained models {#kitti-models}
+
+Model name                                                                                                                                                        | Speed (ms) | Pascal mAP@0.5 | Outputs
+----------------------------------------------------------------------------------------------------------------------------------------------------------------- | :---: | :-------------: | :-----:
+[faster_rcnn_resnet101_kitti](http://download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_kitti_2018_01_28.tar.gz) | 79  | 87              | Boxes
+
+## Open Images-trained models {#open-images-models}
+
+Model name                                                                                                                                                        | Speed (ms) | Open Images mAP@0.5[^2] | Outputs
+----------------------------------------------------------------------------------------------------------------------------------------------------------------- | :---: | :-------------: | :-----:
+[faster_rcnn_inception_resnet_v2_atrous_oid](http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_oid_2018_01_28.tar.gz) | 727 | 37              | Boxes
+[faster_rcnn_inception_resnet_v2_atrous_lowproposals_oid](http://download.tensorflow.org/models/object_detection/faster_rcnn_inception_resnet_v2_atrous_lowproposals_oid_2018_01_28.tar.gz) | 347  |               | Boxes
+
+
+## AVA v2.1 trained models {#ava-models}
+
+Model name                                                                                                                                                        | Speed (ms) | Pascal mAP@0.5 | Outputs
+----------------------------------------------------------------------------------------------------------------------------------------------------------------- | :---: | :-------------: | :-----:
+[faster_rcnn_resnet101_ava_v2.1](http://download.tensorflow.org/models/object_detection/faster_rcnn_resnet101_ava_v2.1_2018_04_30.tar.gz) | 93  | 11              | Boxes
+
+
+[^1]: See [MSCOCO evaluation protocol](http://cocodataset.org/#detections-eval).
+[^2]: This is PASCAL mAP with a slightly different way of true positives computation: see [Open Images evaluation protocol](evaluation_protocols.md#open-images).
+

research/mlperf_object_detection/Mask_RCNN/object_detection/g3doc/evaluation_protocols.md

+# Supported object detection evaluation protocols
+
+The Tensorflow Object Detection API currently supports three evaluation protocols,
+that can be configured in `EvalConfig` by setting `metrics_set` to the
+corresponding value.
+
+## PASCAL VOC 2010 detection metric
+
+`EvalConfig.metrics_set='pascal_voc_detection_metrics'`
+
+The commonly used mAP metric for evaluating the quality of object detectors,
+computed according to the protocol of the PASCAL VOC Challenge 2010-2012. The
+protocol is available
+[here](http://host.robots.ox.ac.uk/pascal/VOC/voc2010/devkit_doc_08-May-2010.pdf).
+
+## Weighted PASCAL VOC detection metric
+
+`EvalConfig.metrics_set='weighted_pascal_voc_detection_metrics'`
+
+The weighted PASCAL metric computes the mean average precision as the average
+precision when treating all classes as a single class. In comparison,
+PASCAL metrics computes the mean average precision as the mean of the
+per-class average precisions.
+
+For example, the test set consists of two classes, "cat" and "dog", and there
+are ten times more boxes of "cat" than those of "dog". According to PASCAL VOC
+2010 metric, performance on each of the two classes would contribute equally
+towards the final mAP value, while for the Weighted PASCAL VOC metric the final
+mAP value will be influenced by frequency of each class.
+
+## PASCAL VOC 2010 instance segmentation metric
+
+`EvalConfig.metrics_set='pascal_voc_instance_segmentation_metrics'`
+
+Similar to Pascal VOC 2010 detection metric, but computes the intersection over
+union based on the object masks instead of object boxes.
+
+## Weighted PASCAL VOC instance segmentation metric
+
+`EvalConfig.metrics_set='weighted_pascal_voc_instance_segmentation_metrics'`
+
+Similar to the weighted pascal voc 2010 detection metric, but computes the
+intersection over union based on the object masks instead of object boxes.
+
+## Open Images V2 detection metric
+
+`EvalConfig.metrics_set='open_images_V2_detection_metrics'`
+
+This metric is defined originally for evaluating detector performance on [Open
+Images V2 dataset](https://github.com/openimages/dataset) and is fairly similar
+to the PASCAL VOC 2010 metric mentioned above. It computes interpolated average
+precision (AP) for each class and averages it among all classes (mAP).
+
+The difference to the PASCAL VOC 2010 metric is the following: Open Images
+annotations contain `group-of` ground-truth boxes (see [Open Images data
+description](https://github.com/openimages/dataset#annotations-human-bboxcsv)),
+that are treated differently for the purpose of deciding whether detections are
+"true positives", "ignored", "false positives". Here we define these three
+cases:
+
+A detection is a "true positive" if there is a non-group-of ground-truth box,
+such that:
+
+*   The detection box and the ground-truth box are of the same class, and
+    intersection-over-union (IoU) between the detection box and the ground-truth
+    box is greater than the IoU threshold (default value 0.5). \
+    Illustration of handling non-group-of boxes: \
+    ![alt
+    groupof_case_eval](img/nongroupof_case_eval.png "illustration of handling non-group-of boxes: yellow box - ground truth bounding box; green box - true positive; red box - false positives.")
+
+    *   yellow box - ground-truth box;
+    *   green box - true positive;
+    *   red boxes - false positives.
+
+*   This is the highest scoring detection for this ground truth box that
+    satisfies the criteria above.
+
+A detection is "ignored" if it is not a true positive, and there is a `group-of`
+ground-truth box such that:
+
+*   The detection box and the ground-truth box are of the same class, and the
+    area of intersection between the detection box and the ground-truth box
+    divided by the area of the detection is greater than 0.5. This is intended
+    to measure whether the detection box is approximately inside the group-of
+    ground-truth box. \
+    Illustration of handling `group-of` boxes: \
+    ![alt
+    groupof_case_eval](img/groupof_case_eval.png "illustration of handling group-of boxes: yellow box - ground truth bounding box; grey boxes - two detections of cars, that are ignored; red box - false positive.")
+
+    *   yellow box - ground-truth box;
+    *   grey boxes - two detections on cars, that are ignored;
+    *   red box - false positive.
+
+A detection is a "false positive" if it is neither a "true positive" nor
+"ignored".
+
+Precision and recall are defined as:
+
+* Precision = number-of-true-positives/(number-of-true-positives + number-of-false-positives)
+* Recall = number-of-true-positives/number-of-non-group-of-boxes
+
+Note that detections ignored as firing on a `group-of` ground-truth box do not
+contribute to the number of true positives.
+
+The labels in Open Images are organized in a
+[hierarchy](https://storage.googleapis.com/openimages/2017_07/bbox_labels_vis/bbox_labels_vis.html).
+Ground-truth bounding-boxes are annotated with the most specific class available
+in the hierarchy. For example, "car" has two children "limousine" and "van". Any
+other kind of car is annotated as "car" (for example, a sedan). Given this
+convention, the evaluation software treats all classes independently, ignoring
+the hierarchy. To achieve high performance values, object detectors should
+output bounding-boxes labelled in the same manner.
+
+## OID Challenge Object Detection Metric 2018
+
+`EvalConfig.metrics_set='oid_challenge_object_detection_metrics'`
+
+The metric for the OID Challenge Object Detection Metric 2018, Object Detection
+track. The description is provided on the [Open Images Challenge
+website](https://storage.googleapis.com/openimages/web/challenge.html).
+
+## OID Challenge Visual Relationship Detection Metric 2018
+
+The metric for the OID Challenge Visual Relationship Detection Metric 2018, Visual
+Relationship Detection track. The description is provided on the [Open Images
+Challenge
+website](https://storage.googleapis.com/openimages/web/challenge.html). Note:
+this is currently a stand-alone metric, that can be used only through the
+`metrics/oid_vrd_challenge_evaluation.py` util.

research/mlperf_object_detection/Mask_RCNN/object_detection/g3doc/exporting_models.md

+# Exporting a trained model for inference
+
+After your model has been trained, you should export it to a Tensorflow
+graph proto. A checkpoint will typically consist of three files:
+
+* model.ckpt-${CHECKPOINT_NUMBER}.data-00000-of-00001
+* model.ckpt-${CHECKPOINT_NUMBER}.index
+* model.ckpt-${CHECKPOINT_NUMBER}.meta
+
+After you've identified a candidate checkpoint to export, run the following
+command from tensorflow/models/research:
+
+``` bash
+# From tensorflow/models/research/
+python object_detection/export_inference_graph.py \
+    --input_type image_tensor \
+    --pipeline_config_path ${PIPELINE_CONFIG_PATH} \
+    --trained_checkpoint_prefix ${TRAIN_PATH} \
+    --output_directory ${EXPORT_DIR}
+```
+
+Afterwards, you should see the directory ${EXPORT_DIR} containing the following:
+
+* output_inference_graph.pb, the frozen graph format of the exported model
+* saved_model/, a directory containing the saved model format of the exported model
+* model.ckpt.*, the model checkpoints used for exporting
+* checkpoint, a file specifying to restore included checkpoint files

research/mlperf_object_detection/Mask_RCNN/object_detection/g3doc/faq.md

+# Frequently Asked Questions
+
+## Q: AttributeError: 'module' object has no attribute 'BackupHandler'
+A: This BackupHandler (tf.contrib.slim.tfexample_decoder.BackupHandler) was
+introduced in tensorflow 1.5.0 so runing with earlier versions may cause this
+issue. It now has been replaced by
+object_detection.data_decoders.tf_example_decoder.BackupHandler. Whoever sees
+this issue should be able to resolve it by syncing your fork to HEAD.
+Same for LookupTensor.
+
+## Q: AttributeError: 'module' object has no attribute 'LookupTensor'
+A: Similar to BackupHandler, syncing your fork to HEAD should make it work.
+
+## Q: Why can't I get the inference time as reported in model zoo?
+A: The inference time reported in model zoo is mean time of testing hundreds of
+images with an internal machine. As mentioned in
+[Tensorflow detection model zoo](detection_model_zoo.md), this speed depends
+highly on one's specific hardware configuration and should be treated more as
+relative timing.