blip2.py 22.8 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3

4
from collections.abc import Iterable, Mapping, Sequence
5
from typing import Annotated, Literal, TypeAlias
6
7
8

import torch
import torch.nn as nn
9
10
11
12
13
14
from transformers import (
    BatchFeature,
    Blip2Config,
    Blip2QFormerConfig,
    apply_chunking_to_forward,
)
15

16
from vllm.config import CacheConfig, VllmConfig
17
from vllm.config.multimodal import BaseDummyOptions
18
19
20
from vllm.model_executor.layers.activation import get_act_fn
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.multimodal import MULTIMODAL_REGISTRY
21
22
23
24
25
from vllm.multimodal.inputs import (
    MultiModalDataDict,
    MultiModalFieldConfig,
    MultiModalKwargsItems,
)
26
from vllm.multimodal.parse import MultiModalDataItems
27
28
29
30
31
32
33
from vllm.multimodal.processing import (
    BaseMultiModalProcessor,
    BaseProcessingInfo,
    PromptIndexTargets,
    PromptInsertion,
    PromptUpdate,
)
34
from vllm.multimodal.profiling import BaseDummyInputsBuilder
35
from vllm.sequence import IntermediateTensors
36
from vllm.utils.tensor_schema import TensorSchema, TensorShape
37
38

from .blip import BlipVisionModel
39
40
41
42
43
44
from .interfaces import (
    MultiModalEmbeddings,
    SupportsMultiModal,
    SupportsPP,
    SupportsQuant,
)
45
from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
46
47


48
49
50
51
52
53
54
55
class Blip2ImagePixelInputs(TensorSchema):
    """
    Dimensions:
        - bn: Batch size * number of images
        - c: Number of channels (3)
        - h: Height of each image
        - w: Width of each image
    """
56

57
    type: Literal["pixel_values"]
58
    data: Annotated[torch.Tensor, TensorShape("bn", 3, "h", "w")]
59
60


61
class Blip2ImageEmbeddingInputs(TensorSchema):
62
    """
63
64
65
66
67
    Dimensions:
        - bn: Batch size * number of images
        - f: Image feature size
        - h: Hidden size (must match the hidden size of language model backbone)
    """
68

69
70
    type: Literal["image_embeds"]
    data: Annotated[torch.Tensor, TensorShape("bn", "f", "h")]
71
72


73
Blip2ImageInputs: TypeAlias = Blip2ImagePixelInputs | Blip2ImageEmbeddingInputs
74

75
76
77
78
79
80

class Blip2QFormerMultiHeadAttention(nn.Module):
    def __init__(
        self,
        config: Blip2QFormerConfig,
        *,
81
82
        quant_config: QuantizationConfig | None,
        cache_config: CacheConfig | None,
83
        is_cross_attention: bool = False,
84
        prefix: str = "",
85
86
87
88
89
90
91
92
93
94
95
96
    ) -> None:
        super().__init__()

        self.config = config

        if config.hidden_size % config.num_attention_heads != 0:
            raise ValueError(
                f"The hidden size ({config.hidden_size}) is not a multiple of "
                f"the number of attention heads ({config.num_attention_heads})"
            )

        self.num_attention_heads = config.num_attention_heads
97
        self.attention_head_size = config.hidden_size // config.num_attention_heads
98
99
100
101
102
103
104
105
106
107
108
        self.all_head_size = self.num_attention_heads * self.attention_head_size
        self.scaling = self.attention_head_size**-0.5

        self.query = nn.Linear(config.hidden_size, self.all_head_size)
        if is_cross_attention:
            kv_hidden_size = config.encoder_hidden_size
        else:
            kv_hidden_size = config.hidden_size
        self.key = nn.Linear(kv_hidden_size, self.all_head_size)
        self.value = nn.Linear(kv_hidden_size, self.all_head_size)

109
110
111
        self.position_embedding_type = getattr(
            config, "position_embedding_type", "absolute"
        )
112
        if self.position_embedding_type != "absolute":
113
114
115
            raise NotImplementedError(
                f"Unsupported position_embedding_type: {self.position_embedding_type}"
            )
116
117
118
119

        self.dropout = nn.Dropout(config.attention_probs_dropout_prob)

    def transpose_for_scores(self, x):
120
        x = x.view(*x.size()[:-1], self.num_attention_heads, self.attention_head_size)
121
122
123
124
125
        return x.permute(0, 2, 1, 3)

    def forward(
        self,
        hidden_states: torch.Tensor,
126
        encoder_hidden_states: torch.FloatTensor | None = None,
127
128
129
130
    ):
        is_cross_attention = encoder_hidden_states is not None

        if is_cross_attention:
131
132
            key_layer = self.transpose_for_scores(self.key(encoder_hidden_states))
            value_layer = self.transpose_for_scores(self.value(encoder_hidden_states))
133
134
135
136
137
138
139
140
        else:
            key_layer = self.transpose_for_scores(self.key(hidden_states))
            value_layer = self.transpose_for_scores(self.value(hidden_states))

        mixed_query_layer = self.query(hidden_states)

        query_layer = self.transpose_for_scores(mixed_query_layer)

141
142
        attention_scores = torch.matmul(query_layer, key_layer.transpose(-1, -2))
        attention_probs = torch.softmax(attention_scores * self.scaling, dim=-1)
143
144
145
146
147
148
149
150

        # This is actually dropping out entire tokens to attend to, which might
        # seem a bit unusual, but is taken from the original Transformer paper.
        attention_probs_dropped = self.dropout(attention_probs)

        context_layer = torch.matmul(attention_probs_dropped, value_layer)

        context_layer = context_layer.permute(0, 2, 1, 3).contiguous()
151
152
153
        context_layer = context_layer.view(
            *context_layer.size()[:-2], self.all_head_size
        )
154
155
156
157
158

        return context_layer


class Blip2QFormerSelfOutput(nn.Module):
159
    def __init__(self, config: Blip2QFormerConfig, prefix: str = "") -> None:
160
161
162
        super().__init__()

        self.dense = nn.Linear(config.hidden_size, config.hidden_size)
163
        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
        self.dropout = nn.Dropout(config.hidden_dropout_prob)

    def forward(
        self,
        hidden_states: torch.Tensor,
        input_tensor: torch.Tensor,
    ) -> torch.Tensor:
        hidden_states = self.dense(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states


class Blip2QFormerAttention(nn.Module):
    def __init__(
        self,
        config: Blip2QFormerConfig,
        *,
182
183
        quant_config: QuantizationConfig | None,
        cache_config: CacheConfig | None,
184
        is_cross_attention: bool = False,
185
        prefix: str = "",
186
187
188
189
190
191
192
193
    ) -> None:
        super().__init__()

        self.attention = Blip2QFormerMultiHeadAttention(
            config,
            quant_config=quant_config,
            cache_config=cache_config,
            is_cross_attention=is_cross_attention,
194
            prefix=f"{prefix}.attention",
195
196
        )

197
        self.output = Blip2QFormerSelfOutput(config, prefix=f"{prefix}.output")
198
199
200
201

    def forward(
        self,
        hidden_states: torch.Tensor,
202
        encoder_hidden_states: torch.FloatTensor | None = None,
203
    ) -> tuple[torch.Tensor]:
204
205
206
207
208
209
210
211
212
213
        self_output = self.attention(
            hidden_states,
            encoder_hidden_states=encoder_hidden_states,
        )
        attention_output = self.output(self_output, hidden_states)

        return attention_output


class Blip2QFormerIntermediate(nn.Module):
214
    def __init__(self, config: Blip2QFormerConfig, prefix: str = "") -> None:
215
216
217
218
219
220
221
222
223
224
225
226
        super().__init__()

        self.dense = nn.Linear(config.hidden_size, config.intermediate_size)
        self.intermediate_act_fn = get_act_fn(config.hidden_act)

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        hidden_states = self.dense(hidden_states)
        hidden_states = self.intermediate_act_fn(hidden_states)
        return hidden_states


class Blip2QFormerOutput(nn.Module):
227
    def __init__(self, config: Blip2QFormerConfig, prefix: str = "") -> None:
228
229
230
        super().__init__()

        self.dense = nn.Linear(config.intermediate_size, config.hidden_size)
231
        self.LayerNorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
        self.dropout = nn.Dropout(config.hidden_dropout_prob)

    def forward(
        self,
        hidden_states: torch.Tensor,
        input_tensor: torch.Tensor,
    ) -> torch.Tensor:
        hidden_states = self.dense(hidden_states)
        hidden_states = self.dropout(hidden_states)
        hidden_states = self.LayerNorm(hidden_states + input_tensor)
        return hidden_states


class Blip2QFormerLayer(nn.Module):
    def __init__(
        self,
        config: Blip2QFormerConfig,
        *,
250
251
        quant_config: QuantizationConfig | None,
        cache_config: CacheConfig | None,
252
        layer_idx: int,
253
        prefix: str = "",
254
255
256
257
258
    ) -> None:
        super().__init__()

        self.chunk_size_feed_forward = config.chunk_size_feed_forward
        self.seq_len_dim = 1
259
260
261
262
263
264
        self.attention = Blip2QFormerAttention(
            config,
            quant_config=quant_config,
            cache_config=cache_config,
            prefix=f"{prefix}.attention",
        )
265
266
267
268
269
270
271
272

        self.layer_idx = layer_idx

        if layer_idx % config.cross_attention_frequency == 0:
            self.crossattention = Blip2QFormerAttention(
                config,
                quant_config=quant_config,
                cache_config=cache_config,
273
                is_cross_attention=True,
274
275
                prefix=f"{prefix}.crossattention",
            )
276
277
278
279
            self.has_cross_attention = True
        else:
            self.has_cross_attention = False

280
        self.intermediate_query = Blip2QFormerIntermediate(
281
282
283
            config, prefix=f"{prefix}.intermediate_query"
        )
        self.output_query = Blip2QFormerOutput(config, prefix=f"{prefix}.output_query")
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315

    def forward(
        self,
        hidden_states: torch.FloatTensor,
        encoder_hidden_states: torch.FloatTensor,
        query_length: int,
    ):
        attention_output = self.attention(hidden_states)

        if query_length > 0:
            query_attention_output = attention_output[:, :query_length, :]

            if self.has_cross_attention:
                query_attention_output = self.crossattention(
                    query_attention_output,
                    encoder_hidden_states=encoder_hidden_states,
                )

            layer_output = apply_chunking_to_forward(
                self.feed_forward_chunk_query,
                self.chunk_size_feed_forward,
                self.seq_len_dim,
                query_attention_output,
            )

            if attention_output.shape[1] > query_length:
                layer_output_text = apply_chunking_to_forward(
                    self.feed_forward_chunk,
                    self.chunk_size_feed_forward,
                    self.seq_len_dim,
                    attention_output[:, query_length:, :],
                )
316
                layer_output = torch.cat([layer_output, layer_output_text], dim=1)
317
318
319
320
321
322
323
324
325
326
        else:
            layer_output = apply_chunking_to_forward(
                self.feed_forward_chunk,
                self.chunk_size_feed_forward,
                self.seq_len_dim,
                attention_output,
            )

        return layer_output

327
    def feed_forward_chunk(self, attention_output: torch.Tensor) -> torch.Tensor:
328
329
330
331
        intermediate_output = self.intermediate(attention_output)
        layer_output = self.output(intermediate_output, attention_output)
        return layer_output

332
    def feed_forward_chunk_query(self, attention_output: torch.Tensor) -> torch.Tensor:
333
334
335
336
337
338
339
340
341
342
        intermediate_output = self.intermediate_query(attention_output)
        layer_output = self.output_query(intermediate_output, attention_output)
        return layer_output


class Blip2QFormerEncoder(nn.Module):
    def __init__(
        self,
        config: Blip2QFormerConfig,
        *,
343
344
        quant_config: QuantizationConfig | None,
        cache_config: CacheConfig | None,
345
        prefix: str = "",
346
347
348
349
350
    ) -> None:
        super().__init__()

        self.config = config

351
352
353
354
355
356
357
358
359
360
361
362
        self.layer = nn.ModuleList(
            [
                Blip2QFormerLayer(
                    config,
                    quant_config=quant_config,
                    cache_config=cache_config,
                    layer_idx=layer_idx,
                    prefix=f"{prefix}.layer.{layer_idx}",
                )
                for layer_idx in range(config.num_hidden_layers)
            ]
        )
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387

    def forward(
        self,
        hidden_states: torch.FloatTensor,
        encoder_hidden_states: torch.FloatTensor,
        query_length: int,
    ) -> torch.Tensor:
        for i in range(self.config.num_hidden_layers):
            layer_module = self.layer[i]

            hidden_states = layer_module(
                hidden_states,
                encoder_hidden_states=encoder_hidden_states,
                query_length=query_length,
            )

        return hidden_states


# Adapted from https://github.com/huggingface/transformers/blob/v4.41.2/src/transformers/models/blip_2/modeling_blip_2.py#L1025
class Blip2QFormerModel(nn.Module):
    def __init__(
        self,
        config: Blip2QFormerConfig,
        *,
388
389
        quant_config: QuantizationConfig | None,
        cache_config: CacheConfig | None,
390
        prefix: str = "",
391
392
393
394
395
    ) -> None:
        super().__init__()

        self.config = config

396
        self.layernorm = nn.LayerNorm(config.hidden_size, eps=config.layer_norm_eps)
397
398
        self.dropout = nn.Dropout(config.hidden_dropout_prob)

399
400
401
402
403
404
        self.encoder = Blip2QFormerEncoder(
            config,
            quant_config=quant_config,
            cache_config=cache_config,
            prefix=f"{prefix}.encoder",
        )
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424

    def forward(
        self,
        query_embeds: torch.FloatTensor,
        encoder_hidden_states: torch.FloatTensor,
    ) -> torch.Tensor:
        query_length = query_embeds.shape[1]

        embedding_output = self.layernorm(query_embeds)
        embedding_output = self.dropout(embedding_output)

        sequence_output = self.encoder(
            embedding_output,
            encoder_hidden_states=encoder_hidden_states,
            query_length=query_length,
        )

        return sequence_output


425
426
class Blip2ProcessingInfo(BaseProcessingInfo):
    def get_hf_config(self):
427
        return self.ctx.get_hf_config(Blip2Config)
428

429
    def get_supported_mm_limits(self) -> Mapping[str, int | None]:
430
431
        return {"image": 1}

432
433
434
435
436
437
    def get_num_image_tokens(self) -> int:
        hf_config = self.get_hf_config()
        return hf_config.num_query_tokens


class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):
438
439
440
441
    def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
        return ""

    def get_dummy_mm_data(
442
443
444
        self,
        seq_len: int,
        mm_counts: Mapping[str, int],
445
        mm_options: Mapping[str, BaseDummyOptions] | None = None,
446
    ) -> MultiModalDataDict:
447
        hf_config = self.info.get_hf_config()
448
449
450
451
452
        vision_config = hf_config.vision_config

        max_image_size = vision_config.image_size
        num_images = mm_counts.get("image", 0)

453
454
        image_overrides = mm_options.get("image") if mm_options else None

455
        return {
456
457
458
459
460
461
            "image": self._get_dummy_images(
                width=max_image_size,
                height=max_image_size,
                num_images=num_images,
                overrides=image_overrides,
            )
462
463
464
        }


465
class Blip2MultiModalProcessor(BaseMultiModalProcessor[Blip2ProcessingInfo]):
466
467
468
469
470
    def _call_hf_processor(
        self,
        prompt: str,
        mm_data: Mapping[str, object],
        mm_kwargs: Mapping[str, object],
471
        tok_kwargs: Mapping[str, object],
472
473
474
475
476
477
478
479
480
481
482
    ) -> BatchFeature:
        if not mm_data:
            # HF processor always adds placeholders even when there's no image
            tokenizer = self.info.get_tokenizer()
            prompt_ids = tokenizer.encode(prompt)
            return BatchFeature(dict(input_ids=[prompt_ids]), tensor_type="pt")

        return super()._call_hf_processor(
            prompt=prompt,
            mm_data=mm_data,
            mm_kwargs=mm_kwargs,
483
            tok_kwargs=tok_kwargs,
484
485
        )

486
487
488
489
490
491
492
493
494
    def _get_mm_fields_config(
        self,
        hf_inputs: BatchFeature,
        hf_processor_mm_kwargs: Mapping[str, object],
    ) -> Mapping[str, MultiModalFieldConfig]:
        return dict(
            pixel_values=MultiModalFieldConfig.batched("image"),
            image_embeds=MultiModalFieldConfig.batched("image"),
        )
495

496
    def _get_prompt_updates(
497
498
499
        self,
        mm_items: MultiModalDataItems,
        hf_processor_mm_kwargs: Mapping[str, object],
500
        out_mm_kwargs: MultiModalKwargsItems,
501
    ) -> Sequence[PromptUpdate]:
502
503
504
        tokenizer = self.info.get_tokenizer()
        vocab = tokenizer.get_vocab()

505
        image_token_id = vocab["<image>"]
506
        num_image_tokens = self.info.get_num_image_tokens()
507
        image_tokens = [image_token_id] * num_image_tokens
508
509

        return [
510
            PromptInsertion(
511
                modality="image",
512
                target=PromptIndexTargets.start(),
513
                insertion=image_tokens,
514
515
            )
        ]
516
517


518
519
520
521
522
523
524
525
@MULTIMODAL_REGISTRY.register_processor(
    Blip2MultiModalProcessor,
    info=Blip2ProcessingInfo,
    dummy_inputs=Blip2DummyInputsBuilder,
)
class Blip2ForConditionalGeneration(
    nn.Module, SupportsMultiModal, SupportsPP, SupportsQuant
):
526
    merge_by_field_config = True
527

528
    @classmethod
529
    def get_placeholder_str(cls, modality: str, i: int) -> str | None:
530
531
532
533
534
        if modality.startswith("image"):
            return None

        raise ValueError("Only image modality is supported")

535
    def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
536
        super().__init__()
537
538
539
540
        config = vllm_config.model_config.hf_config
        cache_config = vllm_config.cache_config
        quant_config = vllm_config.quant_config
        multimodal_config = vllm_config.model_config.multimodal_config
541
542
543
544
        self.config = config
        self.multimodal_config = multimodal_config

        # TODO: Optionally initializes this for supporting embeddings.
545
        self.vision_model = BlipVisionModel(config.vision_config, quant_config)
546
547

        self.query_tokens = nn.Parameter(
548
549
            torch.zeros(1, config.num_query_tokens, config.qformer_config.hidden_size)
        )
550

551
552
553
554
555
556
        self.qformer = Blip2QFormerModel(
            config.qformer_config,
            cache_config=cache_config,
            quant_config=quant_config,
            prefix=f"{prefix}.qformer",
        )
557
558
559
560
561
562
563

        self.language_projection = nn.Linear(
            config.qformer_config.hidden_size,
            config.text_config.hidden_size,
            bias=True,
        )

564
        self.language_model = init_vllm_registered_model(
565
            vllm_config=vllm_config,
566
567
568
            hf_config=config.text_config,
            prefix=maybe_prefix(prefix, "language_model"),
        )
569

570
        self.make_empty_intermediate_tensors = (
571
572
            self.language_model.make_empty_intermediate_tensors
        )
573

574
    def _parse_and_validate_image_input(
575
        self, **kwargs: object
576
    ) -> Blip2ImageInputs | None:
577
        pixel_values = kwargs.pop("pixel_values", None)
578
        image_embeds = kwargs.pop("image_embeds", None)
579

580
        if pixel_values is None and image_embeds is None:
581
582
            return None

583
        if pixel_values is not None:
584
            expected_h = expected_w = self.config.vision_config.image_size
585
586
587
588
589
            return Blip2ImagePixelInputs(
                type="pixel_values",
                data=pixel_values,
                resolve_bindings={"h": expected_h, "w": expected_w},
            )
590
591
592
593

        if image_embeds is not None:
            return Blip2ImageEmbeddingInputs(
                type="image_embeds",
594
                data=image_embeds,
595
596
597
            )

        raise AssertionError("This line should be unreachable.")
598

599
600
601
    def _image_pixels_to_features(
        self, vision_model: BlipVisionModel, pixel_values: torch.Tensor
    ) -> torch.Tensor:
602
603
604
605
606
607
        # NOTE: we skip the step to select the vision feature layer since
        # this is already done inside the vision tower
        image_features = vision_model(pixel_values)

        return image_features

608
    def _process_image_pixels(self, inputs: Blip2ImagePixelInputs) -> torch.Tensor:
609
610
611
612
613
614
        assert self.vision_model is not None

        pixel_values = inputs["data"]

        return self._image_pixels_to_features(self.vision_model, pixel_values)

615
    def _process_image_input(self, image_input: Blip2ImageInputs) -> torch.Tensor:
616
617
618
        if image_input["type"] == "image_embeds":
            return image_input["data"]

619
620
621
        assert self.vision_model is not None
        image_features = self._process_image_pixels(image_input)

622
        query_tokens = self.query_tokens.expand(image_features.shape[0], -1, -1)
623
624
625
626
627
628
629
        query_output = self.qformer(
            query_embeds=query_tokens,
            encoder_hidden_states=image_features,
        )

        return self.language_projection(query_output)

630
631
632
    def get_language_model(self) -> torch.nn.Module:
        return self.language_model

633
    def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
634
635
        image_input = self._parse_and_validate_image_input(**kwargs)
        if image_input is None:
636
            return []
637
638
639
        vision_embeddings = self._process_image_input(image_input)
        return vision_embeddings

640
641
642
643
    def forward(
        self,
        input_ids: torch.Tensor,
        positions: torch.Tensor,
644
645
        intermediate_tensors: IntermediateTensors | None = None,
        inputs_embeds: torch.Tensor | None = None,
646
        **kwargs: object,
647
    ) -> IntermediateTensors:
648
649
650
651
652
653
654
655
656
657
658
659
        """Run forward pass for BLIP-2.

        One key thing to understand is the `input_ids` already accounts for the
        positions of the to-be-inserted image embeddings.

        Concretely, consider a text prompt:
        `"Question: What's the content of the image? Answer:"`.

        Tokenizer outputs:
        `[2, 45641, 35, 653, 18, 5, 1383, 9, 5, 2274, 116, 31652, 35]`.

        To reserve space in KV cache, we have to insert placeholder tokens
660
        before they are inputted to the model, so the input processor prepends
661
662
663
664
665
666
667
668
669
670
671
672
        dummy tokens (denoted as `50265`), resulting in:
        `[50265, ..., 50265, 2, 45641, 35, ..., 31652, 35]`.

        We insert 32 tokens since it corresponds to the number of query
        embeddings outputted by the Q-Former and inputted to the language model.

        This way, the `positions` and `attn_metadata` are consistent
        with the `input_ids`.

        Args:
            input_ids: Flattened (concatenated) input_ids corresponding to a
                batch.
673

674
        Info:
samzong's avatar
samzong committed
675
            [`Blip2ImageInputs`][vllm.model_executor.models.blip2.Blip2ImageInputs]
676
        """
677

678
        if intermediate_tensors is not None:
679
            inputs_embeds = None
680

681
682
683
        hidden_states = self.language_model.model(
            input_ids, positions, intermediate_tensors, inputs_embeds=inputs_embeds
        )
684
685
686

        return hidden_states

687
688
689
    def compute_logits(
        self,
        hidden_states: torch.Tensor,
690
    ) -> torch.Tensor | None:
691
        return self.language_model.compute_logits(hidden_states)
692

693
    def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
694
        loader = AutoWeightsLoader(self)
695
        return loader.load_weights(weights)