loss_ops.py

# Copyright 2022 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

"""Operations for compute losses for centernet."""

import tensorflow as tf

from official.vision.ops import sampling_ops


def _get_shape(tensor, num_dims):
  assert len(tensor.shape.as_list()) == num_dims
  return sampling_ops.combined_static_and_dynamic_shape(tensor)


def flatten_spatial_dimensions(batch_images):
  # pylint: disable=unbalanced-tuple-unpacking
  batch_size, height, width, channels = _get_shape(batch_images, 4)
  return tf.reshape(batch_images, [batch_size, height * width,
                                   channels])


def multi_range(limit,
                value_repetitions=1,
                range_repetitions=1,
                dtype=tf.int32):
  """Creates a sequence with optional value duplication and range repetition.

  As an example (see the Args section for more details),
  _multi_range(limit=2, value_repetitions=3, range_repetitions=4) returns:
  [0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1]
  NOTE: Repurposed from Google OD API.

  Args:
    limit: A 0-D Tensor (scalar). Upper limit of sequence, exclusive.
    value_repetitions: Integer. The number of times a value in the sequence is
      repeated. With value_repetitions=3, the result is [0, 0, 0, 1, 1, 1, ..].
    range_repetitions: Integer. The number of times the range is repeated. With
      range_repetitions=3, the result is [0, 1, 2, .., 0, 1, 2, ..].
    dtype: The type of the elements of the resulting tensor.

  Returns:
    A 1-D tensor of type `dtype` and size
      [`limit` * `value_repetitions` * `range_repetitions`] that contains the
      specified range with given repetitions.
  """
  return tf.reshape(
      tf.tile(
          tf.expand_dims(tf.range(limit, dtype=dtype), axis=-1),
          multiples=[range_repetitions, value_repetitions]), [-1])


def add_batch_to_indices(indices):
  shape = tf.shape(indices)
  batch_size = shape[0]
  num_instances = shape[1]
  batch_range = multi_range(limit=batch_size, value_repetitions=num_instances)
  batch_range = tf.reshape(batch_range, shape=(batch_size, num_instances, 1))

  return tf.concat([batch_range, indices], axis=-1)


def get_num_instances_from_weights(gt_weights_list):
  """Computes the number of instances/boxes from the weights in a batch.

  Args:
    gt_weights_list: A list of float tensors with shape
      [max_num_instances] representing whether there is an actual instance in
      the image (with non-zero value) or is padded to match the
      max_num_instances (with value 0.0). The list represents the batch
      dimension.

  Returns:
    A scalar integer tensor incidating how many instances/boxes are in the
    images in the batch. Note that this function is usually used to normalize
    the loss so the minimum return value is 1 to avoid weird behavior.
  """

  # This can execute in graph mode
  gt_weights_list = tf.convert_to_tensor(
      gt_weights_list, dtype=gt_weights_list[0].dtype)
  num_instances = tf.map_fn(
      fn=lambda x: tf.math.count_nonzero(x, dtype=gt_weights_list[0].dtype),
      elems=gt_weights_list)

  num_instances = tf.reduce_sum(num_instances)
  num_instances = tf.maximum(num_instances, 1)
  return num_instances


def get_batch_predictions_from_indices(batch_predictions, indices):
  """Gets the values of predictions in a batch at the given indices.

  The indices are expected to come from the offset targets generation functions
  in this library. The returned value is intended to be used inside a loss
  function.

  Args:
    batch_predictions: A tensor of shape [batch_size, height, width, channels]
      or [batch_size, height, width, class, channels] for class-specific
      features (e.g. keypoint joint offsets).
    indices: A tensor of shape [num_instances, 3] for single class features or
      [num_instances, 4] for multiple classes features.

  Returns:
    values: A tensor of shape [num_instances, channels] holding the predicted
      values at the given indices.
  """

  return tf.gather_nd(batch_predictions, indices)


def get_valid_anchor_weights_in_flattened_image(true_image_shapes, height,
                                                width):
  """Computes valid anchor weights for an image assuming pixels to be flattened.

  This function is useful when we only want to penalize valid areas in the
  image in the case when padding is used. The function assumes that the loss
  function will be applied after flattening the spatial dimensions and returns
  anchor weights accordingly.

  Args:
    true_image_shapes: An integer tensor of shape [batch_size, 3] representing
      the true image shape (without padding) for each sample in the batch.
    height: height of the prediction from the network.
    width: width of the prediction from the network.

  Returns:
    valid_anchor_weights: a float tensor of shape [batch_size, height * width]
    with 1s in locations where the spatial coordinates fall within the height
    and width in true_image_shapes.
  """

  indices = tf.reshape(tf.range(height * width), [1, -1])
  batch_size = tf.shape(true_image_shapes)[0]
  batch_indices = tf.ones((batch_size, 1), dtype=tf.int32) * indices

  y_coords, x_coords, _ = get_row_col_channel_indices_from_flattened_indices(
      batch_indices, width, 1)

  max_y, max_x = true_image_shapes[:, 0], true_image_shapes[:, 1]
  max_x = tf.cast(tf.expand_dims(max_x, 1), tf.float32)
  max_y = tf.cast(tf.expand_dims(max_y, 1), tf.float32)

  x_coords = tf.cast(x_coords, tf.float32)
  y_coords = tf.cast(y_coords, tf.float32)

  valid_mask = tf.math.logical_and(x_coords < max_x, y_coords < max_y)

  return tf.cast(valid_mask, tf.float32)


def get_row_col_channel_indices_from_flattened_indices(indices: int,
                                                       num_cols: int,
                                                       num_channels: int):
  """Computes row, column and channel indices from flattened indices.

  NOTE: Repurposed from Google OD API.

  Args:
    indices: An `int` tensor of any shape holding the indices in the flattened
      space.
    num_cols: `int`, number of columns in the image (width).
    num_channels: `int`, number of channels in the image.

  Returns:
    row_indices: The row indices corresponding to each of the input indices.
      Same shape as indices.
    col_indices: The column indices corresponding to each of the input indices.
      Same shape as indices.
    channel_indices. The channel indices corresponding to each of the input
      indices.
  """
  # Avoid using mod operator to make the ops more easy to be compatible with
  # different environments, e.g. WASM.

  # all inputs and outputs are dtype int32
  row_indices = (indices // num_channels) // num_cols
  col_indices = (indices // num_channels) - row_indices * num_cols
  channel_indices_temp = indices // num_channels
  channel_indices = indices - channel_indices_temp * num_channels

  return row_indices, col_indices, channel_indices