"mmdet3d/structures/bbox_3d/base_box3d.py" did not exist on "9fcfd78fd418fe3867dded0d13f41b62a4fb000b"
eval_util.py 47.4 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
15
"""Common utility functions for evaluation."""
pkulzc's avatar
pkulzc committed
16
17
18
19
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

20
import collections
21
import os
22
import re
23
24
25
import time

import numpy as np
pkulzc's avatar
pkulzc committed
26
from six.moves import range
27
28
import tensorflow as tf

29
from tensorflow.contrib import slim
30
31
32
33
from object_detection.core import box_list
from object_detection.core import box_list_ops
from object_detection.core import keypoint_ops
from object_detection.core import standard_fields as fields
34
from object_detection.metrics import coco_evaluation
35
from object_detection.protos import eval_pb2
36
from object_detection.utils import label_map_util
37
from object_detection.utils import object_detection_evaluation
38
from object_detection.utils import ops
39
from object_detection.utils import shape_utils
40
41
from object_detection.utils import visualization_utils as vis_utils

42
EVAL_KEYPOINT_METRIC = 'coco_keypoint_metrics'
43

44
45
46
47
48
49
# A dictionary of metric names to classes that implement the metric. The classes
# in the dictionary must implement
# utils.object_detection_evaluation.DetectionEvaluator interface.
EVAL_METRICS_CLASS_DICT = {
    'coco_detection_metrics':
        coco_evaluation.CocoDetectionEvaluator,
50
51
    'coco_keypoint_metrics':
        coco_evaluation.CocoKeypointEvaluator,
52
53
    'coco_mask_metrics':
        coco_evaluation.CocoMaskEvaluator,
54
55
    'oid_challenge_detection_metrics':
        object_detection_evaluation.OpenImagesDetectionChallengeEvaluator,
56
57
58
    'oid_challenge_segmentation_metrics':
        object_detection_evaluation
        .OpenImagesInstanceSegmentationChallengeEvaluator,
59
60
61
62
    'pascal_voc_detection_metrics':
        object_detection_evaluation.PascalDetectionEvaluator,
    'weighted_pascal_voc_detection_metrics':
        object_detection_evaluation.WeightedPascalDetectionEvaluator,
63
64
    'precision_at_recall_detection_metrics':
        object_detection_evaluation.PrecisionAtRecallDetectionEvaluator,
65
66
67
68
69
70
    'pascal_voc_instance_segmentation_metrics':
        object_detection_evaluation.PascalInstanceSegmentationEvaluator,
    'weighted_pascal_voc_instance_segmentation_metrics':
        object_detection_evaluation.WeightedPascalInstanceSegmentationEvaluator,
    'oid_V2_detection_metrics':
        object_detection_evaluation.OpenImagesDetectionEvaluator,
71
72
73
74
}

EVAL_DEFAULT_METRIC = 'coco_detection_metrics'

75
76
77
78
79
80
81
82
83

def write_metrics(metrics, global_step, summary_dir):
  """Write metrics to a summary directory.

  Args:
    metrics: A dictionary containing metric names and values.
    global_step: Global step at which the metrics are computed.
    summary_dir: Directory to write tensorflow summaries to.
  """
84
  tf.logging.info('Writing metrics to tf summary.')
85
  summary_writer = tf.summary.FileWriterCache.get(summary_dir)
86
87
88
89
90
  for key in sorted(metrics):
    summary = tf.Summary(value=[
        tf.Summary.Value(tag=key, simple_value=metrics[key]),
    ])
    summary_writer.add_summary(summary, global_step)
91
92
    tf.logging.info('%s: %f', key, metrics[key])
  tf.logging.info('Metrics written to tf summary.')
93
94


95
# TODO(rathodv): Add tests.
96
97
98
99
100
101
102
103
def visualize_detection_results(result_dict,
                                tag,
                                global_step,
                                categories,
                                summary_dir='',
                                export_dir='',
                                agnostic_mode=False,
                                show_groundtruth=False,
104
                                groundtruth_box_visualization_color='black',
105
                                min_score_thresh=.5,
106
107
108
109
                                max_num_predictions=20,
                                skip_scores=False,
                                skip_labels=False,
                                keep_image_id_for_visualization_export=False):
110
111
112
113
114
115
116
117
118
119
120
121
  """Visualizes detection results and writes visualizations to image summaries.

  This function visualizes an image with its detected bounding boxes and writes
  to image summaries which can be viewed on tensorboard.  It optionally also
  writes images to a directory. In the case of missing entry in the label map,
  unknown class name in the visualization is shown as "N/A".

  Args:
    result_dict: a dictionary holding groundtruth and detection
      data corresponding to each image being evaluated.  The following keys
      are required:
        'original_image': a numpy array representing the image with shape
122
          [1, height, width, 3] or [1, height, width, 1]
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
        'detection_boxes': a numpy array of shape [N, 4]
        'detection_scores': a numpy array of shape [N]
        'detection_classes': a numpy array of shape [N]
      The following keys are optional:
        'groundtruth_boxes': a numpy array of shape [N, 4]
        'groundtruth_keypoints': a numpy array of shape [N, num_keypoints, 2]
      Detections are assumed to be provided in decreasing order of score and for
      display, and we assume that scores are probabilities between 0 and 1.
    tag: tensorboard tag (string) to associate with image.
    global_step: global step at which the visualization are generated.
    categories: a list of dictionaries representing all possible categories.
      Each dict in this list has the following keys:
          'id': (required) an integer id uniquely identifying this category
          'name': (required) string representing category name
            e.g., 'cat', 'dog', 'pizza'
          'supercategory': (optional) string representing the supercategory
            e.g., 'animal', 'vehicle', 'food', etc
    summary_dir: the output directory to which the image summaries are written.
    export_dir: the output directory to which images are written.  If this is
      empty (default), then images are not exported.
    agnostic_mode: boolean (default: False) controlling whether to evaluate in
      class-agnostic mode or not.
    show_groundtruth: boolean (default: False) controlling whether to show
      groundtruth boxes in addition to detected boxes
147
148
    groundtruth_box_visualization_color: box color for visualizing groundtruth
      boxes
149
150
    min_score_thresh: minimum score threshold for a box to be visualized
    max_num_predictions: maximum number of detections to visualize
151
152
153
154
    skip_scores: whether to skip score when drawing a single detection
    skip_labels: whether to skip label when drawing a single detection
    keep_image_id_for_visualization_export: whether to keep image identifier in
      filename when exported to export_dir
155
156
157
158
159
  Raises:
    ValueError: if result_dict does not contain the expected keys (i.e.,
      'original_image', 'detection_boxes', 'detection_scores',
      'detection_classes')
  """
160
161
  detection_fields = fields.DetectionResultFields
  input_fields = fields.InputDataFields
162
  if not set([
163
164
165
166
      input_fields.original_image,
      detection_fields.detection_boxes,
      detection_fields.detection_scores,
      detection_fields.detection_classes,
167
168
  ]).issubset(set(result_dict.keys())):
    raise ValueError('result_dict does not contain all expected keys.')
169
  if show_groundtruth and input_fields.groundtruth_boxes not in result_dict:
170
171
    raise ValueError('If show_groundtruth is enabled, result_dict must contain '
                     'groundtruth_boxes.')
172
  tf.logging.info('Creating detection visualizations.')
173
174
  category_index = label_map_util.create_category_index(categories)

175
  image = np.squeeze(result_dict[input_fields.original_image], axis=0)
176
177
  if image.shape[2] == 1:  # If one channel image, repeat in RGB.
    image = np.tile(image, [1, 1, 3])
178
179
180
181
182
183
184
  detection_boxes = result_dict[detection_fields.detection_boxes]
  detection_scores = result_dict[detection_fields.detection_scores]
  detection_classes = np.int32((result_dict[
      detection_fields.detection_classes]))
  detection_keypoints = result_dict.get(detection_fields.detection_keypoints)
  detection_masks = result_dict.get(detection_fields.detection_masks)
  detection_boundaries = result_dict.get(detection_fields.detection_boundaries)
185
186
187

  # Plot groundtruth underneath detections
  if show_groundtruth:
188
189
    groundtruth_boxes = result_dict[input_fields.groundtruth_boxes]
    groundtruth_keypoints = result_dict.get(input_fields.groundtruth_keypoints)
190
    vis_utils.visualize_boxes_and_labels_on_image_array(
191
192
193
194
195
        image=image,
        boxes=groundtruth_boxes,
        classes=None,
        scores=None,
        category_index=category_index,
196
197
        keypoints=groundtruth_keypoints,
        use_normalized_coordinates=False,
198
199
        max_boxes_to_draw=None,
        groundtruth_box_visualization_color=groundtruth_box_visualization_color)
200
201
202
203
204
205
206
  vis_utils.visualize_boxes_and_labels_on_image_array(
      image,
      detection_boxes,
      detection_classes,
      detection_scores,
      category_index,
      instance_masks=detection_masks,
207
      instance_boundaries=detection_boundaries,
208
209
210
211
      keypoints=detection_keypoints,
      use_normalized_coordinates=False,
      max_boxes_to_draw=max_num_predictions,
      min_score_thresh=min_score_thresh,
212
213
214
      agnostic_mode=agnostic_mode,
      skip_scores=skip_scores,
      skip_labels=skip_labels)
215
216

  if export_dir:
217
218
219
220
221
222
223
    if keep_image_id_for_visualization_export and result_dict[fields.
                                                              InputDataFields()
                                                              .key]:
      export_path = os.path.join(export_dir, 'export-{}-{}.png'.format(
          tag, result_dict[fields.InputDataFields().key]))
    else:
      export_path = os.path.join(export_dir, 'export-{}.png'.format(tag))
224
225
226
    vis_utils.save_image_array_as_png(image, export_path)

  summary = tf.Summary(value=[
227
228
229
230
231
      tf.Summary.Value(
          tag=tag,
          image=tf.Summary.Image(
              encoded_image_string=vis_utils.encode_image_array_as_png_str(
                  image)))
232
  ])
233
  summary_writer = tf.summary.FileWriterCache.get(summary_dir)
234
235
  summary_writer.add_summary(summary, global_step)

236
237
  tf.logging.info('Detection visualizations written to summary with tag %s.',
                  tag)
238
239


240
241
242
243
244
245
246
247
248
def _run_checkpoint_once(tensor_dict,
                         evaluators=None,
                         batch_processor=None,
                         checkpoint_dirs=None,
                         variables_to_restore=None,
                         restore_fn=None,
                         num_batches=1,
                         master='',
                         save_graph=False,
249
                         save_graph_dir='',
250
                         losses_dict=None,
251
252
                         eval_export_path=None,
                         process_metrics_fn=None):
253
  """Evaluates metrics defined in evaluators and returns summaries.
254
255
256
257

  This function loads the latest checkpoint in checkpoint_dirs and evaluates
  all metrics defined in evaluators. The metrics are processed in batch by the
  batch_processor.
258
259
260
261

  Args:
    tensor_dict: a dictionary holding tensors representing a batch of detections
      and corresponding groundtruth annotations.
262
263
264
    evaluators: a list of object of type DetectionEvaluator to be used for
      evaluation. Note that the metric names produced by different evaluators
      must be unique.
265
266
267
268
269
270
271
272
273
274
275
    batch_processor: a function taking four arguments:
      1. tensor_dict: the same tensor_dict that is passed in as the first
        argument to this function.
      2. sess: a tensorflow session
      3. batch_index: an integer representing the index of the batch amongst
        all batches
      By default, batch_processor is None, which defaults to running:
        return sess.run(tensor_dict)
      To skip an image, it suffices to return an empty dictionary in place of
      result_dict.
    checkpoint_dirs: list of directories to load into an EnsembleModel. If it
276
277
      has only one directory, EnsembleModel will not be used --
        a DetectionModel
278
279
280
281
282
283
284
285
286
287
288
289
290
      will be instantiated directly. Not used if restore_fn is set.
    variables_to_restore: None, or a dictionary mapping variable names found in
      a checkpoint to model variables. The dictionary would normally be
      generated by creating a tf.train.ExponentialMovingAverage object and
      calling its variables_to_restore() method. Not used if restore_fn is set.
    restore_fn: None, or a function that takes a tf.Session object and correctly
      restores all necessary variables from the correct checkpoint file. If
      None, attempts to restore from the first directory in checkpoint_dirs.
    num_batches: the number of batches to use for evaluation.
    master: the location of the Tensorflow session.
    save_graph: whether or not the Tensorflow graph is stored as a pbtxt file.
    save_graph_dir: where to store the Tensorflow graph on disk. If save_graph
      is True this must be non-empty.
291
    losses_dict: optional dictionary of scalar detection losses.
292
293
    eval_export_path: Path for saving a json file that contains the detection
      results in json format.
294
295
296
297
298
299
    process_metrics_fn: a callback called with evaluation results after each
      evaluation is done.  It could be used e.g. to back up checkpoints with
      best evaluation scores, or to call an external system to update evaluation
      results in order to drive best hyper-parameter search.  Parameters are:
      int checkpoint_number, Dict[str, ObjectDetectionEvalMetrics] metrics,
      str checkpoint_file path.
300
301
302
303

  Returns:
    global_step: the count of global steps.
    all_evaluator_metrics: A dictionary containing metric names and values.
304
305
306
307
308
309
310
311
312
313
314

  Raises:
    ValueError: if restore_fn is None and checkpoint_dirs doesn't have at least
      one element.
    ValueError: if save_graph is True and save_graph_dir is not defined.
  """
  if save_graph and not save_graph_dir:
    raise ValueError('`save_graph_dir` must be defined.')
  sess = tf.Session(master, graph=tf.get_default_graph())
  sess.run(tf.global_variables_initializer())
  sess.run(tf.local_variables_initializer())
315
  sess.run(tf.tables_initializer())
316
  checkpoint_file = None
317
318
319
320
321
322
323
324
325
326
327
328
329
  if restore_fn:
    restore_fn(sess)
  else:
    if not checkpoint_dirs:
      raise ValueError('`checkpoint_dirs` must have at least one entry.')
    checkpoint_file = tf.train.latest_checkpoint(checkpoint_dirs[0])
    saver = tf.train.Saver(variables_to_restore)
    saver.restore(sess, checkpoint_file)

  if save_graph:
    tf.train.write_graph(sess.graph_def, save_graph_dir, 'eval.pbtxt')

  counters = {'skipped': 0, 'success': 0}
330
  aggregate_result_losses_dict = collections.defaultdict(list)
331
  with slim.queues.QueueRunners(sess):
332
333
334
    try:
      for batch in range(int(num_batches)):
        if (batch + 1) % 100 == 0:
335
336
          tf.logging.info('Running eval ops batch %d/%d', batch + 1,
                          num_batches)
337
338
        if not batch_processor:
          try:
339
340
341
342
            if not losses_dict:
              losses_dict = {}
            result_dict, result_losses_dict = sess.run([tensor_dict,
                                                        losses_dict])
343
344
            counters['success'] += 1
          except tf.errors.InvalidArgumentError:
345
            tf.logging.info('Skipping image')
346
347
348
            counters['skipped'] += 1
            result_dict = {}
        else:
349
350
          result_dict, result_losses_dict = batch_processor(
              tensor_dict, sess, batch, counters, losses_dict=losses_dict)
351
352
        if not result_dict:
          continue
353
354
        for key, value in iter(result_losses_dict.items()):
          aggregate_result_losses_dict[key].append(value)
355
        for evaluator in evaluators:
356
          # TODO(b/65130867): Use image_id tensor once we fix the input data
357
          # decoders to return correct image_id.
358
          # TODO(akuznetsa): result_dict contains batches of images, while
359
          # add_single_ground_truth_image_info expects a single image. Fix
360
          if (isinstance(result_dict, dict) and
361
              fields.InputDataFields.key in result_dict and
362
363
364
365
              result_dict[fields.InputDataFields.key]):
            image_id = result_dict[fields.InputDataFields.key]
          else:
            image_id = batch
366
          evaluator.add_single_ground_truth_image_info(
367
              image_id=image_id, groundtruth_dict=result_dict)
368
          evaluator.add_single_detected_image_info(
369
370
              image_id=image_id, detections_dict=result_dict)
      tf.logging.info('Running eval batches done.')
371
    except tf.errors.OutOfRangeError:
372
      tf.logging.info('Done evaluating -- epoch limit reached')
373
374
    finally:
      # When done, ask the threads to stop.
375
376
      tf.logging.info('# success: %d', counters['success'])
      tf.logging.info('# skipped: %d', counters['skipped'])
377
      all_evaluator_metrics = {}
378
379
380
381
382
383
384
385
      if eval_export_path and eval_export_path is not None:
        for evaluator in evaluators:
          if (isinstance(evaluator, coco_evaluation.CocoDetectionEvaluator) or
              isinstance(evaluator, coco_evaluation.CocoMaskEvaluator)):
            tf.logging.info('Started dumping to json file.')
            evaluator.dump_detections_to_json_file(
                json_output_path=eval_export_path)
            tf.logging.info('Finished dumping to json file.')
386
387
388
389
390
391
392
      for evaluator in evaluators:
        metrics = evaluator.evaluate()
        evaluator.clear()
        if any(key in all_evaluator_metrics for key in metrics):
          raise ValueError('Metric names between evaluators must not collide.')
        all_evaluator_metrics.update(metrics)
      global_step = tf.train.global_step(sess, tf.train.get_global_step())
393
394
395

      for key, value in iter(aggregate_result_losses_dict.items()):
        all_evaluator_metrics['Losses/' + key] = np.mean(value)
396
397
398
399
400
401
402
403
404
      if process_metrics_fn and checkpoint_file:
        m = re.search(r'model.ckpt-(\d+)$', checkpoint_file)
        if not m:
          tf.logging.error('Failed to parse checkpoint number from: %s',
                           checkpoint_file)
        else:
          checkpoint_number = int(m.group(1))
          process_metrics_fn(checkpoint_number, all_evaluator_metrics,
                             checkpoint_file)
405
  sess.close()
406
  return (global_step, all_evaluator_metrics)
407
408


409
# TODO(rathodv): Add tests.
410
411
def repeated_checkpoint_run(tensor_dict,
                            summary_dir,
412
                            evaluators,
413
414
415
416
417
418
419
                            batch_processor=None,
                            checkpoint_dirs=None,
                            variables_to_restore=None,
                            restore_fn=None,
                            num_batches=1,
                            eval_interval_secs=120,
                            max_number_of_evaluations=None,
420
                            max_evaluation_global_step=None,
421
422
                            master='',
                            save_graph=False,
423
                            save_graph_dir='',
424
                            losses_dict=None,
425
426
                            eval_export_path=None,
                            process_metrics_fn=None):
427
428
429
430
431
432
433
434
435
436
437
  """Periodically evaluates desired tensors using checkpoint_dirs or restore_fn.

  This function repeatedly loads a checkpoint and evaluates a desired
  set of tensors (provided by tensor_dict) and hands the resulting numpy
  arrays to a function result_processor which can be used to further
  process/save/visualize the results.

  Args:
    tensor_dict: a dictionary holding tensors representing a batch of detections
      and corresponding groundtruth annotations.
    summary_dir: a directory to write metrics summaries.
438
439
440
    evaluators: a list of object of type DetectionEvaluator to be used for
      evaluation. Note that the metric names produced by different evaluators
      must be unique.
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
    batch_processor: a function taking three arguments:
      1. tensor_dict: the same tensor_dict that is passed in as the first
        argument to this function.
      2. sess: a tensorflow session
      3. batch_index: an integer representing the index of the batch amongst
        all batches
      By default, batch_processor is None, which defaults to running:
        return sess.run(tensor_dict)
    checkpoint_dirs: list of directories to load into a DetectionModel or an
      EnsembleModel if restore_fn isn't set. Also used to determine when to run
      next evaluation. Must have at least one element.
    variables_to_restore: None, or a dictionary mapping variable names found in
      a checkpoint to model variables. The dictionary would normally be
      generated by creating a tf.train.ExponentialMovingAverage object and
      calling its variables_to_restore() method. Not used if restore_fn is set.
    restore_fn: a function that takes a tf.Session object and correctly restores
      all necessary variables from the correct checkpoint file.
    num_batches: the number of batches to use for evaluation.
    eval_interval_secs: the number of seconds between each evaluation run.
    max_number_of_evaluations: the max number of iterations of the evaluation.
      If the value is left as None the evaluation continues indefinitely.
462
    max_evaluation_global_step: global step when evaluation stops.
463
464
465
466
    master: the location of the Tensorflow session.
    save_graph: whether or not the Tensorflow graph is saved as a pbtxt file.
    save_graph_dir: where to save on disk the Tensorflow graph. If store_graph
      is True this must be non-empty.
467
    losses_dict: optional dictionary of scalar detection losses.
468
469
    eval_export_path: Path for saving a json file that contains the detection
      results in json format.
470
471
472
473
474
475
    process_metrics_fn: a callback called with evaluation results after each
      evaluation is done.  It could be used e.g. to back up checkpoints with
      best evaluation scores, or to call an external system to update evaluation
      results in order to drive best hyper-parameter search.  Parameters are:
      int checkpoint_number, Dict[str, ObjectDetectionEvalMetrics] metrics,
      str checkpoint_file path.
476
477
478
479

  Returns:
    metrics: A dictionary containing metric names and values in the latest
      evaluation.
480
481
482
483
484
485
486

  Raises:
    ValueError: if max_num_of_evaluations is not None or a positive number.
    ValueError: if checkpoint_dirs doesn't have at least one element.
  """
  if max_number_of_evaluations and max_number_of_evaluations <= 0:
    raise ValueError(
487
488
489
490
        '`max_number_of_evaluations` must be either None or a positive number.')
  if max_evaluation_global_step and max_evaluation_global_step <= 0:
    raise ValueError(
        '`max_evaluation_global_step` must be either None or positive.')
491
492
493
494
495
496
497
498

  if not checkpoint_dirs:
    raise ValueError('`checkpoint_dirs` must have at least one entry.')

  last_evaluated_model_path = None
  number_of_evaluations = 0
  while True:
    start = time.time()
499
    tf.logging.info('Starting evaluation at ' + time.strftime(
500
        '%Y-%m-%d-%H:%M:%S', time.gmtime()))
501
502
    model_path = tf.train.latest_checkpoint(checkpoint_dirs[0])
    if not model_path:
503
504
      tf.logging.info('No model found in %s. Will try again in %d seconds',
                      checkpoint_dirs[0], eval_interval_secs)
505
    elif model_path == last_evaluated_model_path:
506
507
      tf.logging.info('Found already evaluated checkpoint. Will try again in '
                      '%d seconds', eval_interval_secs)
508
509
    else:
      last_evaluated_model_path = model_path
510
511
512
513
514
515
516
517
518
519
520
521
      global_step, metrics = _run_checkpoint_once(
          tensor_dict,
          evaluators,
          batch_processor,
          checkpoint_dirs,
          variables_to_restore,
          restore_fn,
          num_batches,
          master,
          save_graph,
          save_graph_dir,
          losses_dict=losses_dict,
522
523
          eval_export_path=eval_export_path,
          process_metrics_fn=process_metrics_fn)
524
      write_metrics(metrics, global_step, summary_dir)
525
526
527
528
      if (max_evaluation_global_step and
          global_step >= max_evaluation_global_step):
        tf.logging.info('Finished evaluation!')
        break
529
530
531
532
    number_of_evaluations += 1

    if (max_number_of_evaluations and
        number_of_evaluations >= max_number_of_evaluations):
533
      tf.logging.info('Finished evaluation!')
534
535
536
537
      break
    time_to_next_eval = start + eval_interval_secs - time.time()
    if time_to_next_eval > 0:
      time.sleep(time_to_next_eval)
538
539
540
541

  return metrics


542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
def _scale_box_to_absolute(args):
  boxes, image_shape = args
  return box_list_ops.to_absolute_coordinates(
      box_list.BoxList(boxes), image_shape[0], image_shape[1]).get()


def _resize_detection_masks(args):
  detection_boxes, detection_masks, image_shape = args
  detection_masks_reframed = ops.reframe_box_masks_to_image_masks(
      detection_masks, detection_boxes, image_shape[0], image_shape[1])
  return tf.cast(tf.greater(detection_masks_reframed, 0.5), tf.uint8)


def _resize_groundtruth_masks(args):
  mask, image_shape = args
  mask = tf.expand_dims(mask, 3)
  mask = tf.image.resize_images(
      mask,
      image_shape,
      method=tf.image.ResizeMethod.NEAREST_NEIGHBOR,
      align_corners=True)
  return tf.cast(tf.squeeze(mask, 3), tf.uint8)


def _scale_keypoint_to_absolute(args):
  keypoints, image_shape = args
  return keypoint_ops.scale(keypoints, image_shape[0], image_shape[1])


571
572
573
574
575
576
577
578
579
580
581
582
583
def result_dict_for_single_example(image,
                                   key,
                                   detections,
                                   groundtruth=None,
                                   class_agnostic=False,
                                   scale_to_absolute=False):
  """Merges all detection and groundtruth information for a single example.

  Note that evaluation tools require classes that are 1-indexed, and so this
  function performs the offset. If `class_agnostic` is True, all output classes
  have label 1.

  Args:
584
    image: A single 4D uint8 image tensor of shape [1, H, W, C].
585
586
587
588
589
590
591
592
593
594
595
596
597
    key: A single string tensor identifying the image.
    detections: A dictionary of detections, returned from
      DetectionModel.postprocess().
    groundtruth: (Optional) Dictionary of groundtruth items, with fields:
      'groundtruth_boxes': [num_boxes, 4] float32 tensor of boxes, in
        normalized coordinates.
      'groundtruth_classes': [num_boxes] int64 tensor of 1-indexed classes.
      'groundtruth_area': [num_boxes] float32 tensor of bbox area. (Optional)
      'groundtruth_is_crowd': [num_boxes] int64 tensor. (Optional)
      'groundtruth_difficult': [num_boxes] int64 tensor. (Optional)
      'groundtruth_group_of': [num_boxes] int64 tensor. (Optional)
      'groundtruth_instance_masks': 3D int64 tensor of instance masks
        (Optional).
598
599
      'groundtruth_keypoints': [num_boxes, num_keypoints, 2] float32 tensor with
        keypoints (Optional).
600
601
    class_agnostic: Boolean indicating whether the detections are class-agnostic
      (i.e. binary). Default False.
602
603
604
    scale_to_absolute: Boolean indicating whether boxes and keypoints should be
      scaled to absolute coordinates. Note that for IoU based evaluations, it
      does not matter whether boxes are expressed in absolute or relative
605
606
607
608
609
610
611
612
613
614
615
      coordinates. Default False.

  Returns:
    A dictionary with:
    'original_image': A [1, H, W, C] uint8 image tensor.
    'key': A string tensor with image identifier.
    'detection_boxes': [max_detections, 4] float32 tensor of boxes, in
      normalized or absolute coordinates, depending on the value of
      `scale_to_absolute`.
    'detection_scores': [max_detections] float32 tensor of scores.
    'detection_classes': [max_detections] int64 tensor of 1-indexed classes.
616
617
    'detection_masks': [max_detections, H, W] float32 tensor of binarized
      masks, reframed to full image masks.
618
619
620
621
622
623
624
625
626
627
628
    'groundtruth_boxes': [num_boxes, 4] float32 tensor of boxes, in
      normalized or absolute coordinates, depending on the value of
      `scale_to_absolute`. (Optional)
    'groundtruth_classes': [num_boxes] int64 tensor of 1-indexed classes.
      (Optional)
    'groundtruth_area': [num_boxes] float32 tensor of bbox area. (Optional)
    'groundtruth_is_crowd': [num_boxes] int64 tensor. (Optional)
    'groundtruth_difficult': [num_boxes] int64 tensor. (Optional)
    'groundtruth_group_of': [num_boxes] int64 tensor. (Optional)
    'groundtruth_instance_masks': 3D int64 tensor of instance masks
      (Optional).
629
630
    'groundtruth_keypoints': [num_boxes, num_keypoints, 2] float32 tensor with
      keypoints (Optional).
631
  """
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655

  if groundtruth:
    max_gt_boxes = tf.shape(
        groundtruth[fields.InputDataFields.groundtruth_boxes])[0]
    for gt_key in groundtruth:
      # expand groundtruth dict along the batch dimension.
      groundtruth[gt_key] = tf.expand_dims(groundtruth[gt_key], 0)

  for detection_key in detections:
    detections[detection_key] = tf.expand_dims(
        detections[detection_key][0], axis=0)

  batched_output_dict = result_dict_for_batched_example(
      image,
      tf.expand_dims(key, 0),
      detections,
      groundtruth,
      class_agnostic,
      scale_to_absolute,
      max_gt_boxes=max_gt_boxes)

  exclude_keys = [
      fields.InputDataFields.original_image,
      fields.DetectionResultFields.num_detections,
656
      fields.InputDataFields.num_groundtruth_boxes
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
  ]

  output_dict = {
      fields.InputDataFields.original_image:
          batched_output_dict[fields.InputDataFields.original_image]
  }

  for key in batched_output_dict:
    # remove the batch dimension.
    if key not in exclude_keys:
      output_dict[key] = tf.squeeze(batched_output_dict[key], 0)
  return output_dict


def result_dict_for_batched_example(images,
                                    keys,
                                    detections,
                                    groundtruth=None,
                                    class_agnostic=False,
                                    scale_to_absolute=False,
                                    original_image_spatial_shapes=None,
678
                                    true_image_shapes=None,
679
680
681
682
683
684
                                    max_gt_boxes=None):
  """Merges all detection and groundtruth information for a single example.

  Note that evaluation tools require classes that are 1-indexed, and so this
  function performs the offset. If `class_agnostic` is True, all output classes
  have label 1.
685
686
687
688
  The groundtruth coordinates of boxes/keypoints in 'groundtruth' dictionary are
  normalized relative to the (potentially padded) input image, while the
  coordinates in 'detection' dictionary are normalized relative to the true
  image shape.
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709

  Args:
    images: A single 4D uint8 image tensor of shape [batch_size, H, W, C].
    keys: A [batch_size] string tensor with image identifier.
    detections: A dictionary of detections, returned from
      DetectionModel.postprocess().
    groundtruth: (Optional) Dictionary of groundtruth items, with fields:
      'groundtruth_boxes': [batch_size, max_number_of_boxes, 4] float32 tensor
        of boxes, in normalized coordinates.
      'groundtruth_classes':  [batch_size, max_number_of_boxes] int64 tensor of
        1-indexed classes.
      'groundtruth_area': [batch_size, max_number_of_boxes] float32 tensor of
        bbox area. (Optional)
      'groundtruth_is_crowd':[batch_size, max_number_of_boxes] int64
        tensor. (Optional)
      'groundtruth_difficult': [batch_size, max_number_of_boxes] int64
        tensor. (Optional)
      'groundtruth_group_of': [batch_size, max_number_of_boxes] int64
        tensor. (Optional)
      'groundtruth_instance_masks': 4D int64 tensor of instance
        masks (Optional).
710
711
712
713
      'groundtruth_keypoints': [batch_size, max_number_of_boxes, num_keypoints,
        2] float32 tensor with keypoints (Optional).
      'groundtruth_keypoint_visibilities': [batch_size, max_number_of_boxes,
        num_keypoints] bool tensor with keypoint visibilities (Optional).
714
715
716
717
718
719
720
721
    class_agnostic: Boolean indicating whether the detections are class-agnostic
      (i.e. binary). Default False.
    scale_to_absolute: Boolean indicating whether boxes and keypoints should be
      scaled to absolute coordinates. Note that for IoU based evaluations, it
      does not matter whether boxes are expressed in absolute or relative
      coordinates. Default False.
    original_image_spatial_shapes: A 2D int32 tensor of shape [batch_size, 2]
      used to resize the image. When set to None, the image size is retained.
722
723
    true_image_shapes: A 2D int32 tensor of shape [batch_size, 3]
      containing the size of the unpadded original_image.
724
725
726
727
728
729
730
731
    max_gt_boxes: [batch_size] tensor representing the maximum number of
      groundtruth boxes to pad.

  Returns:
    A dictionary with:
    'original_image': A [batch_size, H, W, C] uint8 image tensor.
    'original_image_spatial_shape': A [batch_size, 2] tensor containing the
      original image sizes.
732
733
    'true_image_shape': A [batch_size, 3] tensor containing the size of
      the unpadded original_image.
734
735
736
737
738
739
740
741
    'key': A [batch_size] string tensor with image identifier.
    'detection_boxes': [batch_size, max_detections, 4] float32 tensor of boxes,
      in normalized or absolute coordinates, depending on the value of
      `scale_to_absolute`.
    'detection_scores': [batch_size, max_detections] float32 tensor of scores.
    'detection_classes': [batch_size, max_detections] int64 tensor of 1-indexed
      classes.
    'detection_masks': [batch_size, max_detections, H, W] float32 tensor of
742
743
744
745
746
      binarized masks, reframed to full image masks. (Optional)
    'detection_keypoints': [batch_size, max_detections, num_keypoints, 2]
      float32 tensor containing keypoint coordinates. (Optional)
    'detection_keypoint_scores': [batch_size, max_detections, num_keypoints]
      float32 tensor containing keypoint scores. (Optional)
747
748
749
750
751
752
753
754
755
756
757
758
759
760
    'num_detections': [batch_size] int64 tensor containing number of valid
      detections.
    'groundtruth_boxes': [batch_size, num_boxes, 4] float32 tensor of boxes, in
      normalized or absolute coordinates, depending on the value of
      `scale_to_absolute`. (Optional)
    'groundtruth_classes': [batch_size, num_boxes] int64 tensor of 1-indexed
      classes. (Optional)
    'groundtruth_area': [batch_size, num_boxes] float32 tensor of bbox
      area. (Optional)
    'groundtruth_is_crowd': [batch_size, num_boxes] int64 tensor. (Optional)
    'groundtruth_difficult': [batch_size, num_boxes] int64 tensor. (Optional)
    'groundtruth_group_of': [batch_size, num_boxes] int64 tensor. (Optional)
    'groundtruth_instance_masks': 4D int64 tensor of instance masks
      (Optional).
761
762
763
764
    'groundtruth_keypoints': [batch_size, num_boxes, num_keypoints, 2] float32
      tensor with keypoints (Optional).
    'groundtruth_keypoint_visibilities': [batch_size, num_boxes, num_keypoints]
      bool tensor with keypoint visibilities (Optional).
765
766
767
768
    'num_groundtruth_boxes': [batch_size] tensor containing the maximum number
      of groundtruth boxes per image.

  Raises:
769
770
771
772
    ValueError: if original_image_spatial_shape is not 2D int32 tensor of shape
      [2].
    ValueError: if true_image_shapes is not 2D int32 tensor of shape
      [3].
773
  """
774
775
  label_id_offset = 1  # Applying label id offset (b/63711816)

776
  input_data_fields = fields.InputDataFields
777
778
779
780
781
782
783
784
785
786
787
  if original_image_spatial_shapes is None:
    original_image_spatial_shapes = tf.tile(
        tf.expand_dims(tf.shape(images)[1:3], axis=0),
        multiples=[tf.shape(images)[0], 1])
  else:
    if (len(original_image_spatial_shapes.shape) != 2 and
        original_image_spatial_shapes.shape[1] != 2):
      raise ValueError(
          '`original_image_spatial_shape` should be a 2D tensor of shape '
          '[batch_size, 2].')

788
789
790
791
792
793
794
795
796
797
  if true_image_shapes is None:
    true_image_shapes = tf.tile(
        tf.expand_dims(tf.shape(images)[1:4], axis=0),
        multiples=[tf.shape(images)[0], 1])
  else:
    if (len(true_image_shapes.shape) != 2
        and true_image_shapes.shape[1] != 3):
      raise ValueError('`true_image_shapes` should be a 2D tensor of '
                       'shape [batch_size, 3].')

798
  output_dict = {
799
800
801
802
      input_data_fields.original_image:
          images,
      input_data_fields.key:
          keys,
803
      input_data_fields.original_image_spatial_shape: (
804
805
806
          original_image_spatial_shapes),
      input_data_fields.true_image_shape:
          true_image_shapes
807
808
809
  }

  detection_fields = fields.DetectionResultFields
810
811
  detection_boxes = detections[detection_fields.detection_boxes]
  detection_scores = detections[detection_fields.detection_scores]
812
813
  num_detections = tf.cast(detections[detection_fields.num_detections],
                           dtype=tf.int32)
814
815
816
817
818

  if class_agnostic:
    detection_classes = tf.ones_like(detection_scores, dtype=tf.int64)
  else:
    detection_classes = (
819
        tf.to_int64(detections[detection_fields.detection_classes]) +
820
        label_id_offset)
821

822
823
  if scale_to_absolute:
    output_dict[detection_fields.detection_boxes] = (
824
825
826
827
        shape_utils.static_or_dynamic_map_fn(
            _scale_box_to_absolute,
            elems=[detection_boxes, original_image_spatial_shapes],
            dtype=tf.float32))
828
829
  else:
    output_dict[detection_fields.detection_boxes] = detection_boxes
830
  output_dict[detection_fields.detection_classes] = detection_classes
831
  output_dict[detection_fields.detection_scores] = detection_scores
832
  output_dict[detection_fields.num_detections] = num_detections
833
834

  if detection_fields.detection_masks in detections:
835
    detection_masks = detections[detection_fields.detection_masks]
836
    # TODO(rathodv): This should be done in model's postprocess
837
    # function ideally.
838
839
840
841
842
843
844
    output_dict[detection_fields.detection_masks] = (
        shape_utils.static_or_dynamic_map_fn(
            _resize_detection_masks,
            elems=[detection_boxes, detection_masks,
                   original_image_spatial_shapes],
            dtype=tf.uint8))

845
  if detection_fields.detection_keypoints in detections:
846
    detection_keypoints = detections[detection_fields.detection_keypoints]
847
848
849
    output_dict[detection_fields.detection_keypoints] = detection_keypoints
    if scale_to_absolute:
      output_dict[detection_fields.detection_keypoints] = (
850
851
852
853
          shape_utils.static_or_dynamic_map_fn(
              _scale_keypoint_to_absolute,
              elems=[detection_keypoints, original_image_spatial_shapes],
              dtype=tf.float32))
854
855
856
857
858
859
    if detection_fields.detection_keypoint_scores in detections:
      output_dict[detection_fields.detection_keypoint_scores] = detections[
          detection_fields.detection_keypoint_scores]
    else:
      output_dict[detection_fields.detection_keypoint_scores] = tf.ones_like(
          detections[detection_fields.detection_keypoints][:, :, :, 0])
860
861

  if groundtruth:
862
863
864
865
866
867
868
    if max_gt_boxes is None:
      if input_data_fields.num_groundtruth_boxes in groundtruth:
        max_gt_boxes = groundtruth[input_data_fields.num_groundtruth_boxes]
      else:
        raise ValueError(
            'max_gt_boxes must be provided when processing batched examples.')

869
    if input_data_fields.groundtruth_instance_masks in groundtruth:
870
      masks = groundtruth[input_data_fields.groundtruth_instance_masks]
871
872
873
874
875
876
      groundtruth[input_data_fields.groundtruth_instance_masks] = (
          shape_utils.static_or_dynamic_map_fn(
              _resize_groundtruth_masks,
              elems=[masks, original_image_spatial_shapes],
              dtype=tf.uint8))

877
    output_dict.update(groundtruth)
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897

    image_shape = tf.cast(tf.shape(images), tf.float32)
    image_height, image_width = image_shape[1], image_shape[2]

    def _scale_box_to_normalized_true_image(args):
      """Scale the box coordinates to be relative to the true image shape."""
      boxes, true_image_shape = args
      true_image_shape = tf.cast(true_image_shape, tf.float32)
      true_height, true_width = true_image_shape[0], true_image_shape[1]
      normalized_window = tf.stack([0.0, 0.0, true_height / image_height,
                                    true_width / image_width])
      return box_list_ops.change_coordinate_frame(
          box_list.BoxList(boxes), normalized_window).get()

    groundtruth_boxes = groundtruth[input_data_fields.groundtruth_boxes]
    groundtruth_boxes = shape_utils.static_or_dynamic_map_fn(
        _scale_box_to_normalized_true_image,
        elems=[groundtruth_boxes, true_image_shapes], dtype=tf.float32)
    output_dict[input_data_fields.groundtruth_boxes] = groundtruth_boxes

898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
    if input_data_fields.groundtruth_keypoints in groundtruth:
      # If groundtruth_keypoints is in the groundtruth dictionary. Update the
      # coordinates to conform with the true image shape.
      def _scale_keypoints_to_normalized_true_image(args):
        """Scale the box coordinates to be relative to the true image shape."""
        keypoints, true_image_shape = args
        true_image_shape = tf.cast(true_image_shape, tf.float32)
        true_height, true_width = true_image_shape[0], true_image_shape[1]
        normalized_window = tf.stack(
            [0.0, 0.0, true_height / image_height, true_width / image_width])
        return keypoint_ops.change_coordinate_frame(keypoints,
                                                    normalized_window)

      groundtruth_keypoints = groundtruth[
          input_data_fields.groundtruth_keypoints]
      groundtruth_keypoints = shape_utils.static_or_dynamic_map_fn(
          _scale_keypoints_to_normalized_true_image,
          elems=[groundtruth_keypoints, true_image_shapes],
          dtype=tf.float32)
      output_dict[
          input_data_fields.groundtruth_keypoints] = groundtruth_keypoints

920
    if scale_to_absolute:
921
      groundtruth_boxes = output_dict[input_data_fields.groundtruth_boxes]
922
      output_dict[input_data_fields.groundtruth_boxes] = (
923
924
925
926
          shape_utils.static_or_dynamic_map_fn(
              _scale_box_to_absolute,
              elems=[groundtruth_boxes, original_image_spatial_shapes],
              dtype=tf.float32))
927
928
929
930
931
932
933
934
      if input_data_fields.groundtruth_keypoints in groundtruth:
        groundtruth_keypoints = output_dict[
            input_data_fields.groundtruth_keypoints]
        output_dict[input_data_fields.groundtruth_keypoints] = (
            shape_utils.static_or_dynamic_map_fn(
                _scale_keypoint_to_absolute,
                elems=[groundtruth_keypoints, original_image_spatial_shapes],
                dtype=tf.float32))
935

936
937
938
939
940
941
    # For class-agnostic models, groundtruth classes all become 1.
    if class_agnostic:
      groundtruth_classes = groundtruth[input_data_fields.groundtruth_classes]
      groundtruth_classes = tf.ones_like(groundtruth_classes, dtype=tf.int64)
      output_dict[input_data_fields.groundtruth_classes] = groundtruth_classes

942
943
    output_dict[input_data_fields.num_groundtruth_boxes] = max_gt_boxes

944
  return output_dict
945
946


947
948
949
950
951
952
953
954
def get_evaluators(eval_config, categories, evaluator_options=None):
  """Returns the evaluator class according to eval_config, valid for categories.

  Args:
    eval_config: An `eval_pb2.EvalConfig`.
    categories: A list of dicts, each of which has the following keys -
        'id': (required) an integer id uniquely identifying this category.
        'name': (required) string representing category name e.g., 'cat', 'dog'.
955
956
        'keypoints': (optional) dict mapping this category's keypoints to unique
          ids.
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
    evaluator_options: A dictionary of metric names (see
      EVAL_METRICS_CLASS_DICT) to `DetectionEvaluator` initialization
      keyword arguments. For example:
      evalator_options = {
        'coco_detection_metrics': {'include_metrics_per_category': True}
      }

  Returns:
    An list of instances of DetectionEvaluator.

  Raises:
    ValueError: if metric is not in the metric class dictionary.
  """
  evaluator_options = evaluator_options or {}
  eval_metric_fn_keys = eval_config.metrics_set
  if not eval_metric_fn_keys:
    eval_metric_fn_keys = [EVAL_DEFAULT_METRIC]
  evaluators_list = []
  for eval_metric_fn_key in eval_metric_fn_keys:
    if eval_metric_fn_key not in EVAL_METRICS_CLASS_DICT:
      raise ValueError('Metric not found: {}'.format(eval_metric_fn_key))
    kwargs_dict = (evaluator_options[eval_metric_fn_key] if eval_metric_fn_key
                   in evaluator_options else {})
    evaluators_list.append(EVAL_METRICS_CLASS_DICT[eval_metric_fn_key](
        categories,
        **kwargs_dict))
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008

  if isinstance(eval_config, eval_pb2.EvalConfig):
    parameterized_metrics = eval_config.parameterized_metric
    for parameterized_metric in parameterized_metrics:
      assert parameterized_metric.HasField('parameterized_metric')
      if parameterized_metric.WhichOneof(
          'parameterized_metric') == EVAL_KEYPOINT_METRIC:
        keypoint_metrics = parameterized_metric.coco_keypoint_metrics
        # Create category to keypoints mapping dict.
        category_keypoints = {}
        class_label = keypoint_metrics.class_label
        category = None
        for cat in categories:
          if cat['name'] == class_label:
            category = cat
            break
        if not category:
          continue
        keypoints_for_this_class = category['keypoints']
        category_keypoints = [{
            'id': keypoints_for_this_class[kp_name], 'name': kp_name
        } for kp_name in keypoints_for_this_class]
        # Create keypoint evaluator for this category.
        evaluators_list.append(EVAL_METRICS_CLASS_DICT[EVAL_KEYPOINT_METRIC](
            category['id'], category_keypoints, class_label,
            keypoint_metrics.keypoint_label_to_sigmas))
1009
1010
1011
1012
  return evaluators_list


def get_eval_metric_ops_for_evaluators(eval_config,
1013
                                       categories,
1014
1015
                                       eval_dict):
  """Returns eval metrics ops to use with `tf.estimator.EstimatorSpec`.
1016
1017

  Args:
1018
    eval_config: An `eval_pb2.EvalConfig`.
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
    categories: A list of dicts, each of which has the following keys -
        'id': (required) an integer id uniquely identifying this category.
        'name': (required) string representing category name e.g., 'cat', 'dog'.
    eval_dict: An evaluation dictionary, returned from
      result_dict_for_single_example().

  Returns:
    A dictionary of metric names to tuple of value_op and update_op that can be
    used as eval metric ops in tf.EstimatorSpec.
  """
  eval_metric_ops = {}
1030
1031
1032
1033
1034
  evaluator_options = evaluator_options_from_eval_config(eval_config)
  evaluators_list = get_evaluators(eval_config, categories, evaluator_options)
  for evaluator in evaluators_list:
    eval_metric_ops.update(evaluator.get_estimator_eval_metric_ops(
        eval_dict))
1035
  return eval_metric_ops
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059


def evaluator_options_from_eval_config(eval_config):
  """Produces a dictionary of evaluation options for each eval metric.

  Args:
    eval_config: An `eval_pb2.EvalConfig`.

  Returns:
    evaluator_options: A dictionary of metric names (see
      EVAL_METRICS_CLASS_DICT) to `DetectionEvaluator` initialization
      keyword arguments. For example:
      evalator_options = {
        'coco_detection_metrics': {'include_metrics_per_category': True}
      }
  """
  eval_metric_fn_keys = eval_config.metrics_set
  evaluator_options = {}
  for eval_metric_fn_key in eval_metric_fn_keys:
    if eval_metric_fn_key in ('coco_detection_metrics', 'coco_mask_metrics'):
      evaluator_options[eval_metric_fn_key] = {
          'include_metrics_per_category': (
              eval_config.include_metrics_per_category)
      }
1060
1061
1062
1063
1064
    elif eval_metric_fn_key == 'precision_at_recall_detection_metrics':
      evaluator_options[eval_metric_fn_key] = {
          'recall_lower_bound': (eval_config.recall_lower_bound),
          'recall_upper_bound': (eval_config.recall_upper_bound)
      }
1065
  return evaluator_options