model_builder.py 40.8 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""A function to build a DetectionModel from configuration."""
17

18
import functools
19
20
21
22
23
24
25
26
27
from object_detection.builders import anchor_generator_builder
from object_detection.builders import box_coder_builder
from object_detection.builders import box_predictor_builder
from object_detection.builders import hyperparams_builder
from object_detection.builders import image_resizer_builder
from object_detection.builders import losses_builder
from object_detection.builders import matcher_builder
from object_detection.builders import post_processing_builder
from object_detection.builders import region_similarity_calculator_builder as sim_calc
28
from object_detection.core import balanced_positive_negative_sampler as sampler
29
from object_detection.core import post_processing
30
from object_detection.core import target_assigner
31
32
from object_detection.meta_architectures import center_net_meta_arch
from object_detection.meta_architectures import context_rcnn_meta_arch
33
34
35
from object_detection.meta_architectures import faster_rcnn_meta_arch
from object_detection.meta_architectures import rfcn_meta_arch
from object_detection.meta_architectures import ssd_meta_arch
36
from object_detection.predictors.heads import mask_head
37
from object_detection.protos import losses_pb2
38
from object_detection.protos import model_pb2
39
from object_detection.utils import label_map_util
40
from object_detection.utils import ops
41
42
43
44
45
46
47
48
49
50
from object_detection.utils import tf_version

## Feature Extractors for TF
## This section conditionally imports different feature extractors based on the
## Tensorflow version.
##
# pylint: disable=g-import-not-at-top
if tf_version.is_tf2():
  from object_detection.models import center_net_hourglass_feature_extractor
  from object_detection.models import center_net_resnet_feature_extractor
51
  from object_detection.models import center_net_resnet_v1_fpn_feature_extractor
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
  from object_detection.models import faster_rcnn_inception_resnet_v2_keras_feature_extractor as frcnn_inc_res_keras
  from object_detection.models import faster_rcnn_resnet_keras_feature_extractor as frcnn_resnet_keras
  from object_detection.models import ssd_resnet_v1_fpn_keras_feature_extractor as ssd_resnet_v1_fpn_keras
  from object_detection.models.ssd_mobilenet_v1_fpn_keras_feature_extractor import SSDMobileNetV1FpnKerasFeatureExtractor
  from object_detection.models.ssd_mobilenet_v1_keras_feature_extractor import SSDMobileNetV1KerasFeatureExtractor
  from object_detection.models.ssd_mobilenet_v2_fpn_keras_feature_extractor import SSDMobileNetV2FpnKerasFeatureExtractor
  from object_detection.models.ssd_mobilenet_v2_keras_feature_extractor import SSDMobileNetV2KerasFeatureExtractor
  from object_detection.predictors import rfcn_keras_box_predictor

if tf_version.is_tf1():
  from object_detection.models import faster_rcnn_inception_resnet_v2_feature_extractor as frcnn_inc_res
  from object_detection.models import faster_rcnn_inception_v2_feature_extractor as frcnn_inc_v2
  from object_detection.models import faster_rcnn_nas_feature_extractor as frcnn_nas
  from object_detection.models import faster_rcnn_pnas_feature_extractor as frcnn_pnas
  from object_detection.models import faster_rcnn_resnet_v1_feature_extractor as frcnn_resnet_v1
  from object_detection.models import ssd_resnet_v1_fpn_feature_extractor as ssd_resnet_v1_fpn
  from object_detection.models import ssd_resnet_v1_ppn_feature_extractor as ssd_resnet_v1_ppn
  from object_detection.models.embedded_ssd_mobilenet_v1_feature_extractor import EmbeddedSSDMobileNetV1FeatureExtractor
  from object_detection.models.ssd_inception_v2_feature_extractor import SSDInceptionV2FeatureExtractor
  from object_detection.models.ssd_mobilenet_v2_fpn_feature_extractor import SSDMobileNetV2FpnFeatureExtractor
  from object_detection.models.ssd_mobilenet_v2_mnasfpn_feature_extractor import SSDMobileNetV2MnasFPNFeatureExtractor
  from object_detection.models.ssd_inception_v3_feature_extractor import SSDInceptionV3FeatureExtractor
  from object_detection.models.ssd_mobilenet_edgetpu_feature_extractor import SSDMobileNetEdgeTPUFeatureExtractor
  from object_detection.models.ssd_mobilenet_v1_feature_extractor import SSDMobileNetV1FeatureExtractor
  from object_detection.models.ssd_mobilenet_v1_fpn_feature_extractor import SSDMobileNetV1FpnFeatureExtractor
  from object_detection.models.ssd_mobilenet_v1_ppn_feature_extractor import SSDMobileNetV1PpnFeatureExtractor
  from object_detection.models.ssd_mobilenet_v2_feature_extractor import SSDMobileNetV2FeatureExtractor
  from object_detection.models.ssd_mobilenet_v3_feature_extractor import SSDMobileNetV3LargeFeatureExtractor
  from object_detection.models.ssd_mobilenet_v3_feature_extractor import SSDMobileNetV3SmallFeatureExtractor
81
82
83
  from object_detection.models.ssd_mobiledet_feature_extractor import SSDMobileDetCPUFeatureExtractor
  from object_detection.models.ssd_mobiledet_feature_extractor import SSDMobileDetDSPFeatureExtractor
  from object_detection.models.ssd_mobiledet_feature_extractor import SSDMobileDetEdgeTPUFeatureExtractor
84
  from object_detection.models.ssd_mobiledet_feature_extractor import SSDMobileDetGPUFeatureExtractor
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
  from object_detection.models.ssd_pnasnet_feature_extractor import SSDPNASNetFeatureExtractor
  from object_detection.predictors import rfcn_box_predictor
# pylint: enable=g-import-not-at-top

if tf_version.is_tf2():
  SSD_KERAS_FEATURE_EXTRACTOR_CLASS_MAP = {
      'ssd_mobilenet_v1_keras': SSDMobileNetV1KerasFeatureExtractor,
      'ssd_mobilenet_v1_fpn_keras': SSDMobileNetV1FpnKerasFeatureExtractor,
      'ssd_mobilenet_v2_keras': SSDMobileNetV2KerasFeatureExtractor,
      'ssd_mobilenet_v2_fpn_keras': SSDMobileNetV2FpnKerasFeatureExtractor,
      'ssd_resnet50_v1_fpn_keras':
          ssd_resnet_v1_fpn_keras.SSDResNet50V1FpnKerasFeatureExtractor,
      'ssd_resnet101_v1_fpn_keras':
          ssd_resnet_v1_fpn_keras.SSDResNet101V1FpnKerasFeatureExtractor,
      'ssd_resnet152_v1_fpn_keras':
          ssd_resnet_v1_fpn_keras.SSDResNet152V1FpnKerasFeatureExtractor,
  }
102

103
104
105
106
107
108
109
110
111
112
  FASTER_RCNN_KERAS_FEATURE_EXTRACTOR_CLASS_MAP = {
      'faster_rcnn_resnet50_keras':
          frcnn_resnet_keras.FasterRCNNResnet50KerasFeatureExtractor,
      'faster_rcnn_resnet101_keras':
          frcnn_resnet_keras.FasterRCNNResnet101KerasFeatureExtractor,
      'faster_rcnn_resnet152_keras':
          frcnn_resnet_keras.FasterRCNNResnet152KerasFeatureExtractor,
      'faster_rcnn_inception_resnet_v2_keras':
      frcnn_inc_res_keras.FasterRCNNInceptionResnetV2KerasFeatureExtractor,
  }
113

114
115
  CENTER_NET_EXTRACTOR_FUNCTION_MAP = {
      'resnet_v2_50': center_net_resnet_feature_extractor.resnet_v2_50,
116
117
118
119
120
      'resnet_v2_101': center_net_resnet_feature_extractor.resnet_v2_101,
      'resnet_v1_50_fpn':
          center_net_resnet_v1_fpn_feature_extractor.resnet_v1_50_fpn,
      'resnet_v1_101_fpn':
          center_net_resnet_v1_fpn_feature_extractor.resnet_v1_101_fpn,
121
122
      'hourglass_104': center_net_hourglass_feature_extractor.hourglass_104,
  }
123

124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
  FEATURE_EXTRACTOR_MAPS = [
      CENTER_NET_EXTRACTOR_FUNCTION_MAP,
      FASTER_RCNN_KERAS_FEATURE_EXTRACTOR_CLASS_MAP,
      SSD_KERAS_FEATURE_EXTRACTOR_CLASS_MAP
  ]

if tf_version.is_tf1():
  SSD_FEATURE_EXTRACTOR_CLASS_MAP = {
      'ssd_inception_v2':
          SSDInceptionV2FeatureExtractor,
      'ssd_inception_v3':
          SSDInceptionV3FeatureExtractor,
      'ssd_mobilenet_v1':
          SSDMobileNetV1FeatureExtractor,
      'ssd_mobilenet_v1_fpn':
          SSDMobileNetV1FpnFeatureExtractor,
      'ssd_mobilenet_v1_ppn':
          SSDMobileNetV1PpnFeatureExtractor,
      'ssd_mobilenet_v2':
          SSDMobileNetV2FeatureExtractor,
      'ssd_mobilenet_v2_fpn':
          SSDMobileNetV2FpnFeatureExtractor,
      'ssd_mobilenet_v2_mnasfpn':
          SSDMobileNetV2MnasFPNFeatureExtractor,
      'ssd_mobilenet_v3_large':
          SSDMobileNetV3LargeFeatureExtractor,
      'ssd_mobilenet_v3_small':
          SSDMobileNetV3SmallFeatureExtractor,
      'ssd_mobilenet_edgetpu':
          SSDMobileNetEdgeTPUFeatureExtractor,
      'ssd_resnet50_v1_fpn':
          ssd_resnet_v1_fpn.SSDResnet50V1FpnFeatureExtractor,
      'ssd_resnet101_v1_fpn':
          ssd_resnet_v1_fpn.SSDResnet101V1FpnFeatureExtractor,
      'ssd_resnet152_v1_fpn':
          ssd_resnet_v1_fpn.SSDResnet152V1FpnFeatureExtractor,
      'ssd_resnet50_v1_ppn':
          ssd_resnet_v1_ppn.SSDResnet50V1PpnFeatureExtractor,
      'ssd_resnet101_v1_ppn':
          ssd_resnet_v1_ppn.SSDResnet101V1PpnFeatureExtractor,
      'ssd_resnet152_v1_ppn':
          ssd_resnet_v1_ppn.SSDResnet152V1PpnFeatureExtractor,
      'embedded_ssd_mobilenet_v1':
          EmbeddedSSDMobileNetV1FeatureExtractor,
      'ssd_pnasnet':
          SSDPNASNetFeatureExtractor,
170
171
172
173
174
175
176
177
      'ssd_mobiledet_cpu':
          SSDMobileDetCPUFeatureExtractor,
      'ssd_mobiledet_dsp':
          SSDMobileDetDSPFeatureExtractor,
      'ssd_mobiledet_edgetpu':
          SSDMobileDetEdgeTPUFeatureExtractor,
      'ssd_mobiledet_gpu':
          SSDMobileDetGPUFeatureExtractor,
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
  }

  FASTER_RCNN_FEATURE_EXTRACTOR_CLASS_MAP = {
      'faster_rcnn_nas':
      frcnn_nas.FasterRCNNNASFeatureExtractor,
      'faster_rcnn_pnas':
      frcnn_pnas.FasterRCNNPNASFeatureExtractor,
      'faster_rcnn_inception_resnet_v2':
      frcnn_inc_res.FasterRCNNInceptionResnetV2FeatureExtractor,
      'faster_rcnn_inception_v2':
      frcnn_inc_v2.FasterRCNNInceptionV2FeatureExtractor,
      'faster_rcnn_resnet50':
      frcnn_resnet_v1.FasterRCNNResnet50FeatureExtractor,
      'faster_rcnn_resnet101':
      frcnn_resnet_v1.FasterRCNNResnet101FeatureExtractor,
      'faster_rcnn_resnet152':
      frcnn_resnet_v1.FasterRCNNResnet152FeatureExtractor,
  }

  FEATURE_EXTRACTOR_MAPS = [
      SSD_FEATURE_EXTRACTOR_CLASS_MAP,
      FASTER_RCNN_FEATURE_EXTRACTOR_CLASS_MAP
  ]
201

202
203
204
205
206
207
208

def _check_feature_extractor_exists(feature_extractor_type):
  feature_extractors = set().union(*FEATURE_EXTRACTOR_MAPS)
  if feature_extractor_type not in feature_extractors:
    raise ValueError('{} is not supported. See `model_builder.py` for features '
                     'extractors compatible with different versions of '
                     'Tensorflow'.format(feature_extractor_type))
209

210

211
212
213
def _build_ssd_feature_extractor(feature_extractor_config,
                                 is_training,
                                 freeze_batchnorm,
214
                                 reuse_weights=None):
215
216
217
218
219
  """Builds a ssd_meta_arch.SSDFeatureExtractor based on config.

  Args:
    feature_extractor_config: A SSDFeatureExtractor proto config from ssd.proto.
    is_training: True if this feature extractor is being built for training.
220
221
222
223
    freeze_batchnorm: Whether to freeze batch norm parameters during
      training or not. When training with a small batch size (e.g. 1), it is
      desirable to freeze batch norm update and use pretrained batch norm
      params.
224
225
226
227
228
229
230
231
232
233
234
    reuse_weights: if the feature extractor should reuse weights.

  Returns:
    ssd_meta_arch.SSDFeatureExtractor based on config.

  Raises:
    ValueError: On invalid feature extractor type.
  """
  feature_type = feature_extractor_config.type
  depth_multiplier = feature_extractor_config.depth_multiplier
  min_depth = feature_extractor_config.min_depth
235
  pad_to_multiple = feature_extractor_config.pad_to_multiple
236
  use_explicit_padding = feature_extractor_config.use_explicit_padding
237
  use_depthwise = feature_extractor_config.use_depthwise
238

239
240
  is_keras = tf_version.is_tf2()
  if is_keras:
241
242
243
244
245
    conv_hyperparams = hyperparams_builder.KerasLayerHyperparams(
        feature_extractor_config.conv_hyperparams)
  else:
    conv_hyperparams = hyperparams_builder.build(
        feature_extractor_config.conv_hyperparams, is_training)
246
247
  override_base_feature_extractor_hyperparams = (
      feature_extractor_config.override_base_feature_extractor_hyperparams)
248

249
  if not is_keras and feature_type not in SSD_FEATURE_EXTRACTOR_CLASS_MAP:
250
251
    raise ValueError('Unknown ssd feature_extractor: {}'.format(feature_type))

252
  if is_keras:
253
254
255
256
    feature_extractor_class = SSD_KERAS_FEATURE_EXTRACTOR_CLASS_MAP[
        feature_type]
  else:
    feature_extractor_class = SSD_FEATURE_EXTRACTOR_CLASS_MAP[feature_type]
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
  kwargs = {
      'is_training':
          is_training,
      'depth_multiplier':
          depth_multiplier,
      'min_depth':
          min_depth,
      'pad_to_multiple':
          pad_to_multiple,
      'use_explicit_padding':
          use_explicit_padding,
      'use_depthwise':
          use_depthwise,
      'override_base_feature_extractor_hyperparams':
          override_base_feature_extractor_hyperparams
  }

274
275
276
277
278
279
  if feature_extractor_config.HasField('replace_preprocessor_with_placeholder'):
    kwargs.update({
        'replace_preprocessor_with_placeholder':
            feature_extractor_config.replace_preprocessor_with_placeholder
    })

pkulzc's avatar
pkulzc committed
280
281
282
  if feature_extractor_config.HasField('num_layers'):
    kwargs.update({'num_layers': feature_extractor_config.num_layers})

283
  if is_keras:
284
285
286
287
288
289
290
291
292
293
294
    kwargs.update({
        'conv_hyperparams': conv_hyperparams,
        'inplace_batchnorm_update': False,
        'freeze_batchnorm': freeze_batchnorm
    })
  else:
    kwargs.update({
        'conv_hyperparams_fn': conv_hyperparams,
        'reuse_weights': reuse_weights,
    })

295

296
297
  if feature_extractor_config.HasField('fpn'):
    kwargs.update({
298
299
300
301
302
303
        'fpn_min_level':
            feature_extractor_config.fpn.min_level,
        'fpn_max_level':
            feature_extractor_config.fpn.max_level,
        'additional_layer_depth':
            feature_extractor_config.fpn.additional_layer_depth,
304
305
    })

306

307
  return feature_extractor_class(**kwargs)
308
309


310
def _build_ssd_model(ssd_config, is_training, add_summaries):
311
312
313
314
315
316
  """Builds an SSD detection model based on the model config.

  Args:
    ssd_config: A ssd.proto object containing the config for the desired
      SSDMetaArch.
    is_training: True if this model is being built for training purposes.
317
    add_summaries: Whether to add tf summaries in the model.
318
319
  Returns:
    SSDMetaArch based on the config.
320

321
322
323
324
325
  Raises:
    ValueError: If ssd_config.type is not recognized (i.e. not registered in
      model_class_map).
  """
  num_classes = ssd_config.num_classes
326
  _check_feature_extractor_exists(ssd_config.feature_extractor.type)
327
328

  # Feature extractor
329
  feature_extractor = _build_ssd_feature_extractor(
330
      feature_extractor_config=ssd_config.feature_extractor,
331
      freeze_batchnorm=ssd_config.freeze_batchnorm,
332
      is_training=is_training)
333
334
335
336
337

  box_coder = box_coder_builder.build(ssd_config.box_coder)
  matcher = matcher_builder.build(ssd_config.matcher)
  region_similarity_calculator = sim_calc.build(
      ssd_config.similarity_calculator)
338
  encode_background_as_zeros = ssd_config.encode_background_as_zeros
339
  negative_class_weight = ssd_config.negative_class_weight
340
341
  anchor_generator = anchor_generator_builder.build(
      ssd_config.anchor_generator)
342
343
  if feature_extractor.is_keras_model:
    ssd_box_predictor = box_predictor_builder.build_keras(
344
        hyperparams_fn=hyperparams_builder.KerasLayerHyperparams,
345
346
347
348
349
350
351
352
353
354
355
356
        freeze_batchnorm=ssd_config.freeze_batchnorm,
        inplace_batchnorm_update=False,
        num_predictions_per_location_list=anchor_generator
        .num_anchors_per_location(),
        box_predictor_config=ssd_config.box_predictor,
        is_training=is_training,
        num_classes=num_classes,
        add_background_class=ssd_config.add_background_class)
  else:
    ssd_box_predictor = box_predictor_builder.build(
        hyperparams_builder.build, ssd_config.box_predictor, is_training,
        num_classes, ssd_config.add_background_class)
357
358
359
360
  image_resizer_fn = image_resizer_builder.build(ssd_config.image_resizer)
  non_max_suppression_fn, score_conversion_fn = post_processing_builder.build(
      ssd_config.post_processing)
  (classification_loss, localization_loss, classification_weight,
361
362
   localization_weight, hard_example_miner, random_example_sampler,
   expected_loss_weights_fn) = losses_builder.build(ssd_config.loss)
363
  normalize_loss_by_num_matches = ssd_config.normalize_loss_by_num_matches
364
  normalize_loc_loss_by_codesize = ssd_config.normalize_loc_loss_by_codesize
365
366
367
368

  equalization_loss_config = ops.EqualizationLossConfig(
      weight=ssd_config.loss.equalization_loss.weight,
      exclude_prefixes=ssd_config.loss.equalization_loss.exclude_prefixes)
369
370
371
372
373

  target_assigner_instance = target_assigner.TargetAssigner(
      region_similarity_calculator,
      matcher,
      box_coder,
374
      negative_class_weight=negative_class_weight)
375

376
  ssd_meta_arch_fn = ssd_meta_arch.SSDMetaArch
377
  kwargs = {}
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394

  return ssd_meta_arch_fn(
      is_training=is_training,
      anchor_generator=anchor_generator,
      box_predictor=ssd_box_predictor,
      box_coder=box_coder,
      feature_extractor=feature_extractor,
      encode_background_as_zeros=encode_background_as_zeros,
      image_resizer_fn=image_resizer_fn,
      non_max_suppression_fn=non_max_suppression_fn,
      score_conversion_fn=score_conversion_fn,
      classification_loss=classification_loss,
      localization_loss=localization_loss,
      classification_loss_weight=classification_weight,
      localization_loss_weight=localization_weight,
      normalize_loss_by_num_matches=normalize_loss_by_num_matches,
      hard_example_miner=hard_example_miner,
395
      target_assigner_instance=target_assigner_instance,
396
      add_summaries=add_summaries,
397
398
      normalize_loc_loss_by_codesize=normalize_loc_loss_by_codesize,
      freeze_batchnorm=ssd_config.freeze_batchnorm,
399
      inplace_batchnorm_update=ssd_config.inplace_batchnorm_update,
400
      add_background_class=ssd_config.add_background_class,
401
      explicit_background_class=ssd_config.explicit_background_class,
402
      random_example_sampler=random_example_sampler,
403
404
405
406
      expected_loss_weights_fn=expected_loss_weights_fn,
      use_confidences_as_targets=ssd_config.use_confidences_as_targets,
      implicit_example_weight=ssd_config.implicit_example_weight,
      equalization_loss_config=equalization_loss_config,
407
408
      return_raw_detections_during_predict=(
          ssd_config.return_raw_detections_during_predict),
409
      **kwargs)
410
411
412


def _build_faster_rcnn_feature_extractor(
413
    feature_extractor_config, is_training, reuse_weights=True,
414
    inplace_batchnorm_update=False):
415
416
417
418
419
420
421
  """Builds a faster_rcnn_meta_arch.FasterRCNNFeatureExtractor based on config.

  Args:
    feature_extractor_config: A FasterRcnnFeatureExtractor proto config from
      faster_rcnn.proto.
    is_training: True if this feature extractor is being built for training.
    reuse_weights: if the feature extractor should reuse weights.
422
423
424
425
426
    inplace_batchnorm_update: Whether to update batch_norm inplace during
      training. This is required for batch norm to work correctly on TPUs. When
      this is false, user must add a control dependency on
      tf.GraphKeys.UPDATE_OPS for train/loss op in order to update the batch
      norm moving average parameters.
427
428
429
430
431
432
433

  Returns:
    faster_rcnn_meta_arch.FasterRCNNFeatureExtractor based on config.

  Raises:
    ValueError: On invalid feature extractor type.
  """
434
435
  if inplace_batchnorm_update:
    raise ValueError('inplace batchnorm updates not supported.')
436
437
438
  feature_type = feature_extractor_config.type
  first_stage_features_stride = (
      feature_extractor_config.first_stage_features_stride)
439
  batch_norm_trainable = feature_extractor_config.batch_norm_trainable
440
441
442
443
444
445
446

  if feature_type not in FASTER_RCNN_FEATURE_EXTRACTOR_CLASS_MAP:
    raise ValueError('Unknown Faster R-CNN feature_extractor: {}'.format(
        feature_type))
  feature_extractor_class = FASTER_RCNN_FEATURE_EXTRACTOR_CLASS_MAP[
      feature_type]
  return feature_extractor_class(
447
      is_training, first_stage_features_stride,
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
      batch_norm_trainable, reuse_weights=reuse_weights)


def _build_faster_rcnn_keras_feature_extractor(
    feature_extractor_config, is_training,
    inplace_batchnorm_update=False):
  """Builds a faster_rcnn_meta_arch.FasterRCNNKerasFeatureExtractor from config.

  Args:
    feature_extractor_config: A FasterRcnnFeatureExtractor proto config from
      faster_rcnn.proto.
    is_training: True if this feature extractor is being built for training.
    inplace_batchnorm_update: Whether to update batch_norm inplace during
      training. This is required for batch norm to work correctly on TPUs. When
      this is false, user must add a control dependency on
      tf.GraphKeys.UPDATE_OPS for train/loss op in order to update the batch
      norm moving average parameters.

  Returns:
    faster_rcnn_meta_arch.FasterRCNNKerasFeatureExtractor based on config.

  Raises:
    ValueError: On invalid feature extractor type.
  """
  if inplace_batchnorm_update:
    raise ValueError('inplace batchnorm updates not supported.')
  feature_type = feature_extractor_config.type
  first_stage_features_stride = (
      feature_extractor_config.first_stage_features_stride)
  batch_norm_trainable = feature_extractor_config.batch_norm_trainable

  if feature_type not in FASTER_RCNN_KERAS_FEATURE_EXTRACTOR_CLASS_MAP:
    raise ValueError('Unknown Faster R-CNN feature_extractor: {}'.format(
        feature_type))
  feature_extractor_class = FASTER_RCNN_KERAS_FEATURE_EXTRACTOR_CLASS_MAP[
      feature_type]
  return feature_extractor_class(
      is_training, first_stage_features_stride,
      batch_norm_trainable)
487
488


489
def _build_faster_rcnn_model(frcnn_config, is_training, add_summaries):
490
491
492
493
494
495
496
  """Builds a Faster R-CNN or R-FCN detection model based on the model config.

  Builds R-FCN model if the second_stage_box_predictor in the config is of type
  `rfcn_box_predictor` else builds a Faster R-CNN model.

  Args:
    frcnn_config: A faster_rcnn.proto object containing the config for the
497
      desired FasterRCNNMetaArch or RFCNMetaArch.
498
    is_training: True if this model is being built for training purposes.
499
    add_summaries: Whether to add tf summaries in the model.
500
501
502

  Returns:
    FasterRCNNMetaArch based on the config.
503

504
505
506
507
508
509
  Raises:
    ValueError: If frcnn_config.type is not recognized (i.e. not registered in
      model_class_map).
  """
  num_classes = frcnn_config.num_classes
  image_resizer_fn = image_resizer_builder.build(frcnn_config.image_resizer)
510
511
  _check_feature_extractor_exists(frcnn_config.feature_extractor.type)
  is_keras = tf_version.is_tf2()
512
513
514
515
516
517
518
519
520

  if is_keras:
    feature_extractor = _build_faster_rcnn_keras_feature_extractor(
        frcnn_config.feature_extractor, is_training,
        inplace_batchnorm_update=frcnn_config.inplace_batchnorm_update)
  else:
    feature_extractor = _build_faster_rcnn_feature_extractor(
        frcnn_config.feature_extractor, is_training,
        inplace_batchnorm_update=frcnn_config.inplace_batchnorm_update)
521

522
  number_of_stages = frcnn_config.number_of_stages
523
524
525
  first_stage_anchor_generator = anchor_generator_builder.build(
      frcnn_config.first_stage_anchor_generator)

526
527
528
529
  first_stage_target_assigner = target_assigner.create_target_assigner(
      'FasterRCNN',
      'proposal',
      use_matmul_gather=frcnn_config.use_matmul_gather_in_matcher)
530
  first_stage_atrous_rate = frcnn_config.first_stage_atrous_rate
531
532
533
534
535
536
537
  if is_keras:
    first_stage_box_predictor_arg_scope_fn = (
        hyperparams_builder.KerasLayerHyperparams(
            frcnn_config.first_stage_box_predictor_conv_hyperparams))
  else:
    first_stage_box_predictor_arg_scope_fn = hyperparams_builder.build(
        frcnn_config.first_stage_box_predictor_conv_hyperparams, is_training)
538
539
540
541
  first_stage_box_predictor_kernel_size = (
      frcnn_config.first_stage_box_predictor_kernel_size)
  first_stage_box_predictor_depth = frcnn_config.first_stage_box_predictor_depth
  first_stage_minibatch_size = frcnn_config.first_stage_minibatch_size
542
543
  use_static_shapes = frcnn_config.use_static_shapes and (
      frcnn_config.use_static_shapes_for_eval or is_training)
544
545
  first_stage_sampler = sampler.BalancedPositiveNegativeSampler(
      positive_fraction=frcnn_config.first_stage_positive_balance_fraction,
546
547
      is_static=(frcnn_config.use_static_balanced_label_sampler and
                 use_static_shapes))
548
  first_stage_max_proposals = frcnn_config.first_stage_max_proposals
549
550
551
552
553
554
555
556
557
558
559
560
561
  if (frcnn_config.first_stage_nms_iou_threshold < 0 or
      frcnn_config.first_stage_nms_iou_threshold > 1.0):
    raise ValueError('iou_threshold not in [0, 1.0].')
  if (is_training and frcnn_config.second_stage_batch_size >
      first_stage_max_proposals):
    raise ValueError('second_stage_batch_size should be no greater than '
                     'first_stage_max_proposals.')
  first_stage_non_max_suppression_fn = functools.partial(
      post_processing.batch_multiclass_non_max_suppression,
      score_thresh=frcnn_config.first_stage_nms_score_threshold,
      iou_thresh=frcnn_config.first_stage_nms_iou_threshold,
      max_size_per_class=frcnn_config.first_stage_max_proposals,
      max_total_size=frcnn_config.first_stage_max_proposals,
Pooya Davoodi's avatar
Pooya Davoodi committed
562
      use_static_shapes=use_static_shapes,
563
      use_partitioned_nms=frcnn_config.use_partitioned_nms_in_first_stage,
Pooya Davoodi's avatar
Pooya Davoodi committed
564
      use_combined_nms=frcnn_config.use_combined_nms_in_first_stage)
565
566
567
568
569
570
571
572
  first_stage_loc_loss_weight = (
      frcnn_config.first_stage_localization_loss_weight)
  first_stage_obj_loss_weight = frcnn_config.first_stage_objectness_loss_weight

  initial_crop_size = frcnn_config.initial_crop_size
  maxpool_kernel_size = frcnn_config.maxpool_kernel_size
  maxpool_stride = frcnn_config.maxpool_stride

573
574
575
576
  second_stage_target_assigner = target_assigner.create_target_assigner(
      'FasterRCNN',
      'detection',
      use_matmul_gather=frcnn_config.use_matmul_gather_in_matcher)
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
  if is_keras:
    second_stage_box_predictor = box_predictor_builder.build_keras(
        hyperparams_builder.KerasLayerHyperparams,
        freeze_batchnorm=False,
        inplace_batchnorm_update=False,
        num_predictions_per_location_list=[1],
        box_predictor_config=frcnn_config.second_stage_box_predictor,
        is_training=is_training,
        num_classes=num_classes)
  else:
    second_stage_box_predictor = box_predictor_builder.build(
        hyperparams_builder.build,
        frcnn_config.second_stage_box_predictor,
        is_training=is_training,
        num_classes=num_classes)
592
  second_stage_batch_size = frcnn_config.second_stage_batch_size
593
594
  second_stage_sampler = sampler.BalancedPositiveNegativeSampler(
      positive_fraction=frcnn_config.second_stage_balance_fraction,
595
596
      is_static=(frcnn_config.use_static_balanced_label_sampler and
                 use_static_shapes))
597
598
599
600
  (second_stage_non_max_suppression_fn, second_stage_score_conversion_fn
  ) = post_processing_builder.build(frcnn_config.second_stage_post_processing)
  second_stage_localization_loss_weight = (
      frcnn_config.second_stage_localization_loss_weight)
601
602
603
  second_stage_classification_loss = (
      losses_builder.build_faster_rcnn_classification_loss(
          frcnn_config.second_stage_classification_loss))
604
605
  second_stage_classification_loss_weight = (
      frcnn_config.second_stage_classification_loss_weight)
606
607
  second_stage_mask_prediction_loss_weight = (
      frcnn_config.second_stage_mask_prediction_loss_weight)
608
609
610
611
612
613
614
615

  hard_example_miner = None
  if frcnn_config.HasField('hard_example_miner'):
    hard_example_miner = losses_builder.build_hard_example_miner(
        frcnn_config.hard_example_miner,
        second_stage_classification_loss_weight,
        second_stage_localization_loss_weight)

616
617
618
  crop_and_resize_fn = (
      ops.matmul_crop_and_resize if frcnn_config.use_matmul_crop_and_resize
      else ops.native_crop_and_resize)
619
620
  clip_anchors_to_image = (
      frcnn_config.clip_anchors_to_image)
621

622
  common_kwargs = {
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
      'is_training':
          is_training,
      'num_classes':
          num_classes,
      'image_resizer_fn':
          image_resizer_fn,
      'feature_extractor':
          feature_extractor,
      'number_of_stages':
          number_of_stages,
      'first_stage_anchor_generator':
          first_stage_anchor_generator,
      'first_stage_target_assigner':
          first_stage_target_assigner,
      'first_stage_atrous_rate':
          first_stage_atrous_rate,
639
      'first_stage_box_predictor_arg_scope_fn':
640
          first_stage_box_predictor_arg_scope_fn,
641
      'first_stage_box_predictor_kernel_size':
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
          first_stage_box_predictor_kernel_size,
      'first_stage_box_predictor_depth':
          first_stage_box_predictor_depth,
      'first_stage_minibatch_size':
          first_stage_minibatch_size,
      'first_stage_sampler':
          first_stage_sampler,
      'first_stage_non_max_suppression_fn':
          first_stage_non_max_suppression_fn,
      'first_stage_max_proposals':
          first_stage_max_proposals,
      'first_stage_localization_loss_weight':
          first_stage_loc_loss_weight,
      'first_stage_objectness_loss_weight':
          first_stage_obj_loss_weight,
      'second_stage_target_assigner':
          second_stage_target_assigner,
      'second_stage_batch_size':
          second_stage_batch_size,
      'second_stage_sampler':
          second_stage_sampler,
663
      'second_stage_non_max_suppression_fn':
664
665
666
          second_stage_non_max_suppression_fn,
      'second_stage_score_conversion_fn':
          second_stage_score_conversion_fn,
667
      'second_stage_localization_loss_weight':
668
          second_stage_localization_loss_weight,
669
      'second_stage_classification_loss':
670
          second_stage_classification_loss,
671
      'second_stage_classification_loss_weight':
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
          second_stage_classification_loss_weight,
      'hard_example_miner':
          hard_example_miner,
      'add_summaries':
          add_summaries,
      'crop_and_resize_fn':
          crop_and_resize_fn,
      'clip_anchors_to_image':
          clip_anchors_to_image,
      'use_static_shapes':
          use_static_shapes,
      'resize_masks':
          frcnn_config.resize_masks,
      'return_raw_detections_during_predict':
          frcnn_config.return_raw_detections_during_predict,
      'output_final_box_features':
          frcnn_config.output_final_box_features
689
  }
690

691
692
693
694
695
  if ((not is_keras and isinstance(second_stage_box_predictor,
                                   rfcn_box_predictor.RfcnBoxPredictor)) or
      (is_keras and
       isinstance(second_stage_box_predictor,
                  rfcn_keras_box_predictor.RfcnKerasBoxPredictor))):
696
697
698
    return rfcn_meta_arch.RFCNMetaArch(
        second_stage_rfcn_box_predictor=second_stage_box_predictor,
        **common_kwargs)
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
  elif frcnn_config.HasField('context_config'):
    context_config = frcnn_config.context_config
    common_kwargs.update({
        'attention_bottleneck_dimension':
            context_config.attention_bottleneck_dimension,
        'attention_temperature':
            context_config.attention_temperature
    })
    return context_rcnn_meta_arch.ContextRCNNMetaArch(
        initial_crop_size=initial_crop_size,
        maxpool_kernel_size=maxpool_kernel_size,
        maxpool_stride=maxpool_stride,
        second_stage_mask_rcnn_box_predictor=second_stage_box_predictor,
        second_stage_mask_prediction_loss_weight=(
            second_stage_mask_prediction_loss_weight),
        **common_kwargs)
715
716
717
718
719
720
  else:
    return faster_rcnn_meta_arch.FasterRCNNMetaArch(
        initial_crop_size=initial_crop_size,
        maxpool_kernel_size=maxpool_kernel_size,
        maxpool_stride=maxpool_stride,
        second_stage_mask_rcnn_box_predictor=second_stage_box_predictor,
721
722
        second_stage_mask_prediction_loss_weight=(
            second_stage_mask_prediction_loss_weight),
723
        **common_kwargs)
724
725
726
727
728
729
730
731
732

EXPERIMENTAL_META_ARCH_BUILDER_MAP = {
}


def _build_experimental_model(config, is_training, add_summaries=True):
  return EXPERIMENTAL_META_ARCH_BUILDER_MAP[config.name](
      is_training, add_summaries)

733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780

# The class ID in the groundtruth/model architecture is usually 0-based while
# the ID in the label map is 1-based. The offset is used to convert between the
# the two.
CLASS_ID_OFFSET = 1
KEYPOINT_STD_DEV_DEFAULT = 1.0


def keypoint_proto_to_params(kp_config, keypoint_map_dict):
  """Converts CenterNet.KeypointEstimation proto to parameter namedtuple."""
  label_map_item = keypoint_map_dict[kp_config.keypoint_class_name]

  classification_loss, localization_loss, _, _, _, _, _ = (
      losses_builder.build(kp_config.loss))

  keypoint_indices = [
      keypoint.id for keypoint in label_map_item.keypoints
  ]
  keypoint_labels = [
      keypoint.label for keypoint in label_map_item.keypoints
  ]
  keypoint_std_dev_dict = {
      label: KEYPOINT_STD_DEV_DEFAULT for label in keypoint_labels
  }
  if kp_config.keypoint_label_to_std:
    for label, value in kp_config.keypoint_label_to_std.items():
      keypoint_std_dev_dict[label] = value
  keypoint_std_dev = [keypoint_std_dev_dict[label] for label in keypoint_labels]
  return center_net_meta_arch.KeypointEstimationParams(
      task_name=kp_config.task_name,
      class_id=label_map_item.id - CLASS_ID_OFFSET,
      keypoint_indices=keypoint_indices,
      classification_loss=classification_loss,
      localization_loss=localization_loss,
      keypoint_labels=keypoint_labels,
      keypoint_std_dev=keypoint_std_dev,
      task_loss_weight=kp_config.task_loss_weight,
      keypoint_regression_loss_weight=kp_config.keypoint_regression_loss_weight,
      keypoint_heatmap_loss_weight=kp_config.keypoint_heatmap_loss_weight,
      keypoint_offset_loss_weight=kp_config.keypoint_offset_loss_weight,
      heatmap_bias_init=kp_config.heatmap_bias_init,
      keypoint_candidate_score_threshold=(
          kp_config.keypoint_candidate_score_threshold),
      num_candidates_per_keypoint=kp_config.num_candidates_per_keypoint,
      peak_max_pool_kernel_size=kp_config.peak_max_pool_kernel_size,
      unmatched_keypoint_score=kp_config.unmatched_keypoint_score,
      box_scale=kp_config.box_scale,
      candidate_search_scale=kp_config.candidate_search_scale,
781
782
783
      candidate_ranking_mode=kp_config.candidate_ranking_mode,
      offset_peak_radius=kp_config.offset_peak_radius,
      per_keypoint_offset=kp_config.per_keypoint_offset)
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817


def object_detection_proto_to_params(od_config):
  """Converts CenterNet.ObjectDetection proto to parameter namedtuple."""
  loss = losses_pb2.Loss()
  # Add dummy classification loss to avoid the loss_builder throwing error.
  # TODO(yuhuic): update the loss builder to take the classification loss
  # directly.
  loss.classification_loss.weighted_sigmoid.CopyFrom(
      losses_pb2.WeightedSigmoidClassificationLoss())
  loss.localization_loss.CopyFrom(od_config.localization_loss)
  _, localization_loss, _, _, _, _, _ = (losses_builder.build(loss))
  return center_net_meta_arch.ObjectDetectionParams(
      localization_loss=localization_loss,
      scale_loss_weight=od_config.scale_loss_weight,
      offset_loss_weight=od_config.offset_loss_weight,
      task_loss_weight=od_config.task_loss_weight)


def object_center_proto_to_params(oc_config):
  """Converts CenterNet.ObjectCenter proto to parameter namedtuple."""
  loss = losses_pb2.Loss()
  # Add dummy localization loss to avoid the loss_builder throwing error.
  # TODO(yuhuic): update the loss builder to take the localization loss
  # directly.
  loss.localization_loss.weighted_l2.CopyFrom(
      losses_pb2.WeightedL2LocalizationLoss())
  loss.classification_loss.CopyFrom(oc_config.classification_loss)
  classification_loss, _, _, _, _, _, _ = (losses_builder.build(loss))
  return center_net_meta_arch.ObjectCenterParams(
      classification_loss=classification_loss,
      object_center_loss_weight=oc_config.object_center_loss_weight,
      heatmap_bias_init=oc_config.heatmap_bias_init,
      min_box_overlap_iou=oc_config.min_box_overlap_iou,
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
      max_box_predictions=oc_config.max_box_predictions,
      use_labeled_classes=oc_config.use_labeled_classes)


def mask_proto_to_params(mask_config):
  """Converts CenterNet.MaskEstimation proto to parameter namedtuple."""
  loss = losses_pb2.Loss()
  # Add dummy localization loss to avoid the loss_builder throwing error.
  loss.localization_loss.weighted_l2.CopyFrom(
      losses_pb2.WeightedL2LocalizationLoss())
  loss.classification_loss.CopyFrom(mask_config.classification_loss)
  classification_loss, _, _, _, _, _, _ = (losses_builder.build(loss))
  return center_net_meta_arch.MaskParams(
      classification_loss=classification_loss,
      task_loss_weight=mask_config.task_loss_weight,
      mask_height=mask_config.mask_height,
      mask_width=mask_config.mask_width,
      score_threshold=mask_config.score_threshold,
      heatmap_bias_init=mask_config.heatmap_bias_init)
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885


def _build_center_net_model(center_net_config, is_training, add_summaries):
  """Build a CenterNet detection model.

  Args:
    center_net_config: A CenterNet proto object with model configuration.
    is_training: True if this model is being built for training purposes.
    add_summaries: Whether to add tf summaries in the model.

  Returns:
    CenterNetMetaArch based on the config.

  """

  image_resizer_fn = image_resizer_builder.build(
      center_net_config.image_resizer)
  _check_feature_extractor_exists(center_net_config.feature_extractor.type)
  feature_extractor = _build_center_net_feature_extractor(
      center_net_config.feature_extractor)
  object_center_params = object_center_proto_to_params(
      center_net_config.object_center_params)

  object_detection_params = None
  if center_net_config.HasField('object_detection_task'):
    object_detection_params = object_detection_proto_to_params(
        center_net_config.object_detection_task)

  keypoint_params_dict = None
  if center_net_config.keypoint_estimation_task:
    label_map_proto = label_map_util.load_labelmap(
        center_net_config.keypoint_label_map_path)
    keypoint_map_dict = {
        item.name: item for item in label_map_proto.item if item.keypoints
    }
    keypoint_params_dict = {}
    keypoint_class_id_set = set()
    all_keypoint_indices = []
    for task in center_net_config.keypoint_estimation_task:
      kp_params = keypoint_proto_to_params(task, keypoint_map_dict)
      keypoint_params_dict[task.task_name] = kp_params
      all_keypoint_indices.extend(kp_params.keypoint_indices)
      if kp_params.class_id in keypoint_class_id_set:
        raise ValueError(('Multiple keypoint tasks map to the same class id is '
                          'not allowed: %d' % kp_params.class_id))
      else:
        keypoint_class_id_set.add(kp_params.class_id)
    if len(all_keypoint_indices) > len(set(all_keypoint_indices)):
      raise ValueError('Some keypoint indices are used more than once.')
886
887
888
889
890

  mask_params = None
  if center_net_config.HasField('mask_estimation_task'):
    mask_params = mask_proto_to_params(center_net_config.mask_estimation_task)

891
892
893
894
895
896
897
898
  return center_net_meta_arch.CenterNetMetaArch(
      is_training=is_training,
      add_summaries=add_summaries,
      num_classes=center_net_config.num_classes,
      feature_extractor=feature_extractor,
      image_resizer_fn=image_resizer_fn,
      object_center_params=object_center_params,
      object_detection_params=object_detection_params,
899
900
      keypoint_params_dict=keypoint_params_dict,
      mask_params=mask_params)
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918


def _build_center_net_feature_extractor(
    feature_extractor_config):
  """Build a CenterNet feature extractor from the given config."""

  if feature_extractor_config.type not in CENTER_NET_EXTRACTOR_FUNCTION_MAP:
    raise ValueError('\'{}\' is not a known CenterNet feature extractor type'
                     .format(feature_extractor_config.type))

  return CENTER_NET_EXTRACTOR_FUNCTION_MAP[feature_extractor_config.type](
      channel_means=list(feature_extractor_config.channel_means),
      channel_stds=list(feature_extractor_config.channel_stds),
      bgr_ordering=feature_extractor_config.bgr_ordering
  )


META_ARCH_BUILDER_MAP = {
919
920
    'ssd': _build_ssd_model,
    'faster_rcnn': _build_faster_rcnn_model,
921
922
    'experimental_model': _build_experimental_model,
    'center_net': _build_center_net_model
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
}


def build(model_config, is_training, add_summaries=True):
  """Builds a DetectionModel based on the model config.

  Args:
    model_config: A model.proto object containing the config for the desired
      DetectionModel.
    is_training: True if this model is being built for training purposes.
    add_summaries: Whether to add tensorflow summaries in the model graph.
  Returns:
    DetectionModel based on the config.

  Raises:
    ValueError: On invalid meta architecture or model.
  """
  if not isinstance(model_config, model_pb2.DetectionModel):
    raise ValueError('model_config not of type model_pb2.DetectionModel.')

  meta_architecture = model_config.WhichOneof('model')

945
  if meta_architecture not in META_ARCH_BUILDER_MAP:
946
947
    raise ValueError('Unknown meta architecture: {}'.format(meta_architecture))
  else:
948
    build_func = META_ARCH_BUILDER_MAP[meta_architecture]
949
950
    return build_func(getattr(model_config, meta_architecture), is_training,
                      add_summaries)