configuration_detr.py 12.9 KB
Newer Older
NielsRogge's avatar
NielsRogge committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
# coding=utf-8
# Copyright 2021 Facebook AI Research and The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
Sylvain Gugger's avatar
Sylvain Gugger committed
15
""" DETR model configuration"""
NielsRogge's avatar
NielsRogge committed
16

17
import copy
regisss's avatar
regisss committed
18
from collections import OrderedDict
19
from typing import Dict, Mapping
regisss's avatar
regisss committed
20
21
22

from packaging import version

NielsRogge's avatar
NielsRogge committed
23
from ...configuration_utils import PretrainedConfig
regisss's avatar
regisss committed
24
from ...onnx import OnnxConfig
NielsRogge's avatar
NielsRogge committed
25
from ...utils import logging
26
from ..auto import CONFIG_MAPPING
NielsRogge's avatar
NielsRogge committed
27
28
29
30
31
32
33
34
35
36
37
38


logger = logging.get_logger(__name__)

DETR_PRETRAINED_CONFIG_ARCHIVE_MAP = {
    "facebook/detr-resnet-50": "https://huggingface.co/facebook/detr-resnet-50/resolve/main/config.json",
    # See all DETR models at https://huggingface.co/models?filter=detr
}


class DetrConfig(PretrainedConfig):
    r"""
Sylvain Gugger's avatar
Sylvain Gugger committed
39
40
41
42
    This is the configuration class to store the configuration of a [`DetrModel`]. It is used to instantiate a DETR
    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
    defaults will yield a similar configuration to that of the DETR
    [facebook/detr-resnet-50](https://huggingface.co/facebook/detr-resnet-50) architecture.
NielsRogge's avatar
NielsRogge committed
43

Sylvain Gugger's avatar
Sylvain Gugger committed
44
45
    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.
NielsRogge's avatar
NielsRogge committed
46
47

    Args:
48
49
50
51
52
53
        use_timm_backbone (`bool`, *optional*, defaults to `True`):
            Whether or not to use the `timm` library for the backbone. If set to `False`, will use the [`AutoBackbone`]
            API.
        backbone_config (`PretrainedConfig` or `dict`, *optional*):
            The configuration of the backbone model. Only used in case `use_timm_backbone` is set to `False` in which
            case it will default to `ResNetConfig()`.
54
55
        num_channels (`int`, *optional*, defaults to 3):
            The number of input channels.
56
        num_queries (`int`, *optional*, defaults to 100):
Sylvain Gugger's avatar
Sylvain Gugger committed
57
58
            Number of object queries, i.e. detection slots. This is the maximal number of objects [`DetrModel`] can
            detect in a single image. For COCO, we recommend 100 queries.
59
        d_model (`int`, *optional*, defaults to 256):
NielsRogge's avatar
NielsRogge committed
60
            Dimension of the layers.
61
        encoder_layers (`int`, *optional*, defaults to 6):
NielsRogge's avatar
NielsRogge committed
62
            Number of encoder layers.
63
        decoder_layers (`int`, *optional*, defaults to 6):
NielsRogge's avatar
NielsRogge committed
64
            Number of decoder layers.
65
        encoder_attention_heads (`int`, *optional*, defaults to 8):
NielsRogge's avatar
NielsRogge committed
66
            Number of attention heads for each attention layer in the Transformer encoder.
67
        decoder_attention_heads (`int`, *optional*, defaults to 8):
NielsRogge's avatar
NielsRogge committed
68
            Number of attention heads for each attention layer in the Transformer decoder.
69
        decoder_ffn_dim (`int`, *optional*, defaults to 2048):
NielsRogge's avatar
NielsRogge committed
70
            Dimension of the "intermediate" (often named feed-forward) layer in decoder.
71
        encoder_ffn_dim (`int`, *optional*, defaults to 2048):
NielsRogge's avatar
NielsRogge committed
72
            Dimension of the "intermediate" (often named feed-forward) layer in decoder.
73
        activation_function (`str` or `function`, *optional*, defaults to `"relu"`):
Sylvain Gugger's avatar
Sylvain Gugger committed
74
75
            The non-linear activation function (function or string) in the encoder and pooler. If string, `"gelu"`,
            `"relu"`, `"silu"` and `"gelu_new"` are supported.
76
        dropout (`float`, *optional*, defaults to 0.1):
NielsRogge's avatar
NielsRogge committed
77
            The dropout probability for all fully connected layers in the embeddings, encoder, and pooler.
78
        attention_dropout (`float`, *optional*, defaults to 0.0):
NielsRogge's avatar
NielsRogge committed
79
            The dropout ratio for the attention probabilities.
80
        activation_dropout (`float`, *optional*, defaults to 0.0):
NielsRogge's avatar
NielsRogge committed
81
            The dropout ratio for activations inside the fully connected layer.
82
        init_std (`float`, *optional*, defaults to 0.02):
NielsRogge's avatar
NielsRogge committed
83
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
84
        init_xavier_std (`float`, *optional*, defaults to 1):
NielsRogge's avatar
NielsRogge committed
85
            The scaling factor used for the Xavier initialization gain in the HM Attention map module.
Juyoung Kim's avatar
Juyoung Kim committed
86
        encoder_layerdrop (`float`, *optional*, defaults to 0.0):
Sylvain Gugger's avatar
Sylvain Gugger committed
87
88
            The LayerDrop probability for the encoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
            for more details.
Juyoung Kim's avatar
Juyoung Kim committed
89
        decoder_layerdrop (`float`, *optional*, defaults to 0.0):
Sylvain Gugger's avatar
Sylvain Gugger committed
90
91
            The LayerDrop probability for the decoder. See the [LayerDrop paper](see https://arxiv.org/abs/1909.11556)
            for more details.
92
        auxiliary_loss (`bool`, *optional*, defaults to `False`):
NielsRogge's avatar
NielsRogge committed
93
            Whether auxiliary decoding losses (loss at each decoder layer) are to be used.
94
        position_embedding_type (`str`, *optional*, defaults to `"sine"`):
Sylvain Gugger's avatar
Sylvain Gugger committed
95
            Type of position embeddings to be used on top of the image features. One of `"sine"` or `"learned"`.
96
        backbone (`str`, *optional*, defaults to `"resnet50"`):
97
98
            Name of convolutional backbone to use in case `use_timm_backbone` = `True`. Supports any convolutional
            backbone from the timm package. For a list of all available models, see [this
Sylvain Gugger's avatar
Sylvain Gugger committed
99
            page](https://rwightman.github.io/pytorch-image-models/#load-a-pretrained-model).
100
        use_pretrained_backbone (`bool`, *optional*, defaults to `True`):
101
            Whether to use pretrained weights for the backbone. Only supported when `use_timm_backbone` = `True`.
102
        dilation (`bool`, *optional*, defaults to `False`):
103
104
            Whether to replace stride with dilation in the last convolutional block (DC5). Only supported when
            `use_timm_backbone` = `True`.
105
        class_cost (`float`, *optional*, defaults to 1):
NielsRogge's avatar
NielsRogge committed
106
            Relative weight of the classification error in the Hungarian matching cost.
107
        bbox_cost (`float`, *optional*, defaults to 5):
NielsRogge's avatar
NielsRogge committed
108
            Relative weight of the L1 error of the bounding box coordinates in the Hungarian matching cost.
109
        giou_cost (`float`, *optional*, defaults to 2):
NielsRogge's avatar
NielsRogge committed
110
            Relative weight of the generalized IoU loss of the bounding box in the Hungarian matching cost.
111
        mask_loss_coefficient (`float`, *optional*, defaults to 1):
NielsRogge's avatar
NielsRogge committed
112
            Relative weight of the Focal loss in the panoptic segmentation loss.
113
        dice_loss_coefficient (`float`, *optional*, defaults to 1):
NielsRogge's avatar
NielsRogge committed
114
            Relative weight of the DICE/F-1 loss in the panoptic segmentation loss.
115
        bbox_loss_coefficient (`float`, *optional*, defaults to 5):
NielsRogge's avatar
NielsRogge committed
116
            Relative weight of the L1 bounding box loss in the object detection loss.
117
        giou_loss_coefficient (`float`, *optional*, defaults to 2):
NielsRogge's avatar
NielsRogge committed
118
            Relative weight of the generalized IoU loss in the object detection loss.
119
        eos_coefficient (`float`, *optional*, defaults to 0.1):
NielsRogge's avatar
NielsRogge committed
120
121
            Relative classification weight of the 'no-object' class in the object detection loss.

122
    Examples:
NielsRogge's avatar
NielsRogge committed
123

124
    ```python
125
    >>> from transformers import DetrConfig, DetrModel
NielsRogge's avatar
NielsRogge committed
126

127
128
    >>> # Initializing a DETR facebook/detr-resnet-50 style configuration
    >>> configuration = DetrConfig()
NielsRogge's avatar
NielsRogge committed
129

130
    >>> # Initializing a model (with random weights) from the facebook/detr-resnet-50 style configuration
131
    >>> model = DetrModel(configuration)
NielsRogge's avatar
NielsRogge committed
132

133
134
135
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```"""
NielsRogge's avatar
NielsRogge committed
136
137
    model_type = "detr"
    keys_to_ignore_at_inference = ["past_key_values"]
138
139
140
141
    attribute_map = {
        "hidden_size": "d_model",
        "num_attention_heads": "encoder_attention_heads",
    }
NielsRogge's avatar
NielsRogge committed
142
143
144

    def __init__(
        self,
145
146
        use_timm_backbone=True,
        backbone_config=None,
147
        num_channels=3,
NielsRogge's avatar
NielsRogge committed
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
        num_queries=100,
        encoder_layers=6,
        encoder_ffn_dim=2048,
        encoder_attention_heads=8,
        decoder_layers=6,
        decoder_ffn_dim=2048,
        decoder_attention_heads=8,
        encoder_layerdrop=0.0,
        decoder_layerdrop=0.0,
        is_encoder_decoder=True,
        activation_function="relu",
        d_model=256,
        dropout=0.1,
        attention_dropout=0.0,
        activation_dropout=0.0,
        init_std=0.02,
        init_xavier_std=1.0,
        auxiliary_loss=False,
        position_embedding_type="sine",
        backbone="resnet50",
168
        use_pretrained_backbone=True,
NielsRogge's avatar
NielsRogge committed
169
170
171
172
173
174
175
176
177
        dilation=False,
        class_cost=1,
        bbox_cost=5,
        giou_cost=2,
        mask_loss_coefficient=1,
        dice_loss_coefficient=1,
        bbox_loss_coefficient=5,
        giou_loss_coefficient=2,
        eos_coefficient=0.1,
178
        **kwargs,
NielsRogge's avatar
NielsRogge committed
179
    ):
180
181
182
183
184
185
186
187
188
189
190
        if backbone_config is not None and use_timm_backbone:
            raise ValueError("You can't specify both `backbone_config` and `use_timm_backbone`.")

        if not use_timm_backbone:
            if backbone_config is None:
                logger.info("`backbone_config` is `None`. Initializing the config with the default `ResNet` backbone.")
                backbone_config = CONFIG_MAPPING["resnet"](out_features=["stage4"])
            elif isinstance(backbone_config, dict):
                backbone_model_type = backbone_config.get("model_type")
                config_class = CONFIG_MAPPING[backbone_model_type]
                backbone_config = config_class.from_dict(backbone_config)
191
192
            # set timm attributes to None
            dilation, backbone, use_pretrained_backbone = None, None, None
193
194
195

        self.use_timm_backbone = use_timm_backbone
        self.backbone_config = backbone_config
196
        self.num_channels = num_channels
NielsRogge's avatar
NielsRogge committed
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
        self.num_queries = num_queries
        self.d_model = d_model
        self.encoder_ffn_dim = encoder_ffn_dim
        self.encoder_layers = encoder_layers
        self.encoder_attention_heads = encoder_attention_heads
        self.decoder_ffn_dim = decoder_ffn_dim
        self.decoder_layers = decoder_layers
        self.decoder_attention_heads = decoder_attention_heads
        self.dropout = dropout
        self.attention_dropout = attention_dropout
        self.activation_dropout = activation_dropout
        self.activation_function = activation_function
        self.init_std = init_std
        self.init_xavier_std = init_xavier_std
        self.encoder_layerdrop = encoder_layerdrop
        self.decoder_layerdrop = decoder_layerdrop
        self.num_hidden_layers = encoder_layers
        self.auxiliary_loss = auxiliary_loss
        self.position_embedding_type = position_embedding_type
        self.backbone = backbone
217
        self.use_pretrained_backbone = use_pretrained_backbone
NielsRogge's avatar
NielsRogge committed
218
219
220
221
222
223
224
225
226
227
228
        self.dilation = dilation
        # Hungarian matcher
        self.class_cost = class_cost
        self.bbox_cost = bbox_cost
        self.giou_cost = giou_cost
        # Loss coefficients
        self.mask_loss_coefficient = mask_loss_coefficient
        self.dice_loss_coefficient = dice_loss_coefficient
        self.bbox_loss_coefficient = bbox_loss_coefficient
        self.giou_loss_coefficient = giou_loss_coefficient
        self.eos_coefficient = eos_coefficient
229
        super().__init__(is_encoder_decoder=is_encoder_decoder, **kwargs)
NielsRogge's avatar
NielsRogge committed
230
231
232
233
234
235
236
237

    @property
    def num_attention_heads(self) -> int:
        return self.encoder_attention_heads

    @property
    def hidden_size(self) -> int:
        return self.d_model
regisss's avatar
regisss committed
238

239
240
241
    @classmethod
    def from_backbone_config(cls, backbone_config: PretrainedConfig, **kwargs):
        """Instantiate a [`DetrConfig`] (or a derived class) from a pre-trained backbone model configuration.
242

243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
        Args:
            backbone_config ([`PretrainedConfig`]):
                The backbone configuration.
        Returns:
            [`DetrConfig`]: An instance of a configuration object
        """
        return cls(backbone_config=backbone_config, **kwargs)

    def to_dict(self) -> Dict[str, any]:
        """
        Serializes this instance to a Python dictionary. Override the default [`~PretrainedConfig.to_dict`]. Returns:
            `Dict[str, any]`: Dictionary of all the attributes that make up this configuration instance,
        """
        output = copy.deepcopy(self.__dict__)
        if output["backbone_config"] is not None:
            output["backbone_config"] = self.backbone_config.to_dict()
        output["model_type"] = self.__class__.model_type
        return output

regisss's avatar
regisss committed
262
263
264
265
266
267
268
269

class DetrOnnxConfig(OnnxConfig):
    torch_onnx_minimum_version = version.parse("1.11")

    @property
    def inputs(self) -> Mapping[str, Mapping[int, str]]:
        return OrderedDict(
            [
270
                ("pixel_values", {0: "batch", 1: "num_channels", 2: "height", 3: "width"}),
271
                ("pixel_mask", {0: "batch"}),
regisss's avatar
regisss committed
272
273
274
275
276
277
278
279
280
281
            ]
        )

    @property
    def atol_for_validation(self) -> float:
        return 1e-5

    @property
    def default_onnx_opset(self) -> int:
        return 12