conftest.py 43.4 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
5
6
7
8
9
10
11
12

# ruff: noqa

from tblib import pickling_support

# Install support for pickling exceptions so that we can nicely propagate
# failures from tests running in a subprocess.
# This should be run before any custom exception subclasses are defined.
pickling_support.install()

13
import http.server
14
import json
15
import math
16
import mimetypes
17
import os
18
import socket
19
import tempfile
20
21
import threading
from collections.abc import Generator
22
from contextlib import nullcontext
23
from enum import Enum
24
from typing import Any, Callable, Optional, TypedDict, TypeVar, Union, cast
Woosuk Kwon's avatar
Woosuk Kwon committed
25

26
import numpy as np
Woosuk Kwon's avatar
Woosuk Kwon committed
27
28
import pytest
import torch
29
import torch.nn as nn
30
import torch.nn.functional as F
31
from huggingface_hub import snapshot_download
32
from PIL import Image
33
34
from transformers import (AutoConfig, AutoModelForCausalLM, AutoTokenizer,
                          BatchEncoding, BatchFeature)
35
from transformers.models.auto.auto_factory import _BaseAutoModelClass
Woosuk Kwon's avatar
Woosuk Kwon committed
36

37
38
from tests.models.utils import (TokensTextLogprobs,
                                TokensTextLogprobsPromptLogprobs)
Woosuk Kwon's avatar
Woosuk Kwon committed
39
from vllm import LLM, SamplingParams
40
from vllm.assets.audio import AudioAsset
41
from vllm.assets.image import ImageAsset
42
from vllm.assets.video import VideoAsset
43
44
from vllm.config.model import (ConvertOption, RunnerOption,
                               _get_and_verify_dtype)
45
from vllm.connections import global_http_connection
46
from vllm.distributed import (cleanup_dist_env_and_memory,
47
48
                              init_distributed_environment,
                              initialize_model_parallel)
49
from vllm.inputs import TextPrompt
50
from vllm.logger import init_logger
51
from vllm.multimodal.utils import fetch_image
52
from vllm.outputs import RequestOutput
53
from vllm.sampling_params import BeamSearchParams
54
from vllm.sequence import Logprob
55
from vllm.transformers_utils.utils import maybe_model_redirect
56
from vllm.utils import set_default_torch_num_threads
57

58
logger = init_logger(__name__)
Woosuk Kwon's avatar
Woosuk Kwon committed
59

60
61
62
_TEST_DIR = os.path.dirname(__file__)
_TEST_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "example.txt")]
_LONG_PROMPTS = [os.path.join(_TEST_DIR, "prompts", "summary.txt")]
63
_SYS_MSG = os.path.join(_TEST_DIR, "system_messages", "sonnet3.5_nov2024.txt")
64

Cyrus Leung's avatar
Cyrus Leung committed
65
_M = TypeVar("_M")
66

67
_PromptMultiModalInput = Union[list[_M], list[list[_M]]]
Cyrus Leung's avatar
Cyrus Leung committed
68
69

PromptImageInput = _PromptMultiModalInput[Image.Image]
70
PromptAudioInput = _PromptMultiModalInput[tuple[np.ndarray, int]]
Cyrus Leung's avatar
Cyrus Leung committed
71
PromptVideoInput = _PromptMultiModalInput[np.ndarray]
72

73

74
def _read_prompts(filename: str) -> list[str]:
75
    with open(filename) as f:
76
77
        prompts = f.readlines()
        return prompts
Woosuk Kwon's avatar
Woosuk Kwon committed
78
79


80
class ImageAssetPrompts(TypedDict):
81
82
    stop_sign: str
    cherry_blossom: str
83
84


85
class ImageTestAssets(list[ImageAsset]):
86
87

    def __init__(self) -> None:
88
89
90
91
        super().__init__([
            ImageAsset("stop_sign"),
            ImageAsset("cherry_blossom"),
        ])
92

93
    def prompts(self, prompts: ImageAssetPrompts) -> list[str]:
94
95
96
97
98
99
        """
        Convenience method to define the prompt for each test image.

        The order of the returned prompts matches the order of the
        assets when iterating through this object.
        """
100
        return [prompts["stop_sign"], prompts["cherry_blossom"]]
101
102


103
104
class VideoAssetPrompts(TypedDict):
    baby_reading: str
105
106


107
class VideoTestAssets(list[VideoAsset]):
108
109
110

    def __init__(self) -> None:
        super().__init__([
111
            VideoAsset("baby_reading"),
112
113
        ])

114
115
    def prompts(self, prompts: VideoAssetPrompts) -> list[str]:
        return [prompts["baby_reading"]]
116
117


118
class AudioAssetPrompts(TypedDict):
119
120
121
122
    mary_had_lamb: str
    winning_call: str


123
class AudioTestAssets(list[AudioAsset]):
124
125
126
127
128
129
130

    def __init__(self) -> None:
        super().__init__([
            AudioAsset("mary_had_lamb"),
            AudioAsset("winning_call"),
        ])

131
    def prompts(self, prompts: AudioAssetPrompts) -> list[str]:
132
133
        return [prompts["mary_had_lamb"], prompts["winning_call"]]

134

135
IMAGE_ASSETS = ImageTestAssets()
136
"""Singleton instance of {class}`ImageTestAssets`."""
137
VIDEO_ASSETS = VideoTestAssets()
138
"""Singleton instance of {class}`VideoTestAssets`."""
139
AUDIO_ASSETS = AudioTestAssets()
140
"""Singleton instance of {class}`AudioTestAssets`."""
141
142


143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
@pytest.fixture(scope="function", autouse=True)
def cleanup_VLLM_USE_V1(monkeypatch):
    """
    The V1 oracle sets "VLLM_USE_V1" during loading. This means
    that each invocation of a test change the env variable.

    If we touch "VLLM_USE_V1" with monkeypatch, then any changes
    made during the test run by vLLM will be cleaned up.

    This fixture is used by every test.
    """

    # If VLLM_USE_V1 is not set, set then delete. This will
    # cause monkeypatch to clean up VLLM_USE_V1 upon exit
    # if VLLM modifies the value of envs.VLLM_USE_V1.
    if "VLLM_USE_V1" not in os.environ:
        monkeypatch.setenv("VLLM_USE_V1", "")
        monkeypatch.delenv("VLLM_USE_V1")


Joe Runde's avatar
Joe Runde committed
163
@pytest.fixture(params=[True, False])
164
def run_with_both_engines(request, monkeypatch):
Joe Runde's avatar
Joe Runde committed
165
166
167
    # Automatically runs tests twice, once with V1 and once without
    use_v1 = request.param
    # Tests decorated with `@skip_v1` are only run without v1
168
    skip_v0 = request.node.get_closest_marker("skip_v0")
Joe Runde's avatar
Joe Runde committed
169
170
171
172
173
    skip_v1 = request.node.get_closest_marker("skip_v1")

    if use_v1:
        if skip_v1:
            pytest.skip("Skipping test on vllm V1")
174
        monkeypatch.setenv('VLLM_USE_V1', '1')
Joe Runde's avatar
Joe Runde committed
175
    else:
176
177
        if skip_v0:
            pytest.skip("Skipping test on vllm V0")
178
179
180
        monkeypatch.setenv('VLLM_USE_V1', '0')

    yield
Joe Runde's avatar
Joe Runde committed
181
182


183
184
185
186
187
188
189
@pytest.fixture(autouse=True)
def init_test_http_connection():
    # pytest_asyncio may use a different event loop per test
    # so we need to make sure the async client is created anew
    global_http_connection.reuse_client = False


190
191
192
193
194
195
196
197
198
199
200
201
@pytest.fixture
def dist_init():
    temp_file = tempfile.mkstemp()[1]
    init_distributed_environment(
        world_size=1,
        rank=0,
        distributed_init_method=f"file://{temp_file}",
        local_rank=0,
        backend="nccl",
    )
    initialize_model_parallel(1, 1)
    yield
202
    cleanup_dist_env_and_memory()
203
204


205
@pytest.fixture()
206
def should_do_global_cleanup_after_test(request) -> bool:
207
208
209
210
    """Allow subdirectories to skip global cleanup by overriding this fixture.
    This can provide a ~10x speedup for non-GPU unit tests since they don't need
    to initialize torch.
    """
211

212
    return not request.node.get_closest_marker("skip_global_cleanup")
213
214


215
@pytest.fixture(autouse=True)
216
def cleanup_fixture(should_do_global_cleanup_after_test: bool):
217
    yield
218
    if should_do_global_cleanup_after_test:
219
        cleanup_dist_env_and_memory()
220
221


222
223
224
225
226
227
@pytest.fixture(autouse=True)
def dynamo_reset():
    yield
    torch._dynamo.reset()


Woosuk Kwon's avatar
Woosuk Kwon committed
228
@pytest.fixture
229
def example_prompts() -> list[str]:
230
231
    prompts = []
    for filename in _TEST_PROMPTS:
232
        prompts += _read_prompts(filename)
233
234
235
    return prompts


236
237
238
239
240
241
@pytest.fixture
def example_system_message() -> str:
    with open(_SYS_MSG) as f:
        return f.read()


242
243
244
245
246
247
248
class DecoderPromptType(Enum):
    """For encoder/decoder models only."""
    CUSTOM = 1
    NONE = 2
    EMPTY_STR = 3


249
@pytest.fixture
250
def example_long_prompts() -> list[str]:
251
252
    prompts = []
    for filename in _LONG_PROMPTS:
253
        prompts += _read_prompts(filename)
254
    return prompts
Woosuk Kwon's avatar
Woosuk Kwon committed
255
256


257
@pytest.fixture(scope="session")
258
def image_assets() -> ImageTestAssets:
259
260
261
    return IMAGE_ASSETS


262
@pytest.fixture(scope="session")
263
def video_assets() -> VideoTestAssets:
264
265
266
    return VIDEO_ASSETS


267
@pytest.fixture(scope="session")
268
def audio_assets() -> AudioTestAssets:
269
270
271
    return AUDIO_ASSETS


272
_T = TypeVar("_T", nn.Module, torch.Tensor, BatchEncoding, BatchFeature, dict)
273
_R = TypeVar("_R")
274

Woosuk Kwon's avatar
Woosuk Kwon committed
275
276
277

class HfRunner:

278
    def get_default_device(self):
279
        from vllm.platforms import current_platform
280

281
282
        return ("cpu"
                if current_platform.is_cpu() else current_platform.device_type)
283
284

    def wrap_device(self, x: _T, device: Optional[str] = None) -> _T:
285
286
287
        if x is None or isinstance(x, (bool, )):
            return x

288
        if device is None:
289
            device = self.device
290

291
292
        if isinstance(x, dict):
            return {k: self.wrap_device(v, device) for k, v in x.items()}
293

294
295
296
297
        if hasattr(x, "device") and x.device.type == device:
            return x

        return x.to(device)
298

Woosuk Kwon's avatar
Woosuk Kwon committed
299
300
301
    def __init__(
        self,
        model_name: str,
302
        dtype: str = "auto",
303
        *,
304
        model_kwargs: Optional[dict[str, Any]] = None,
305
        trust_remote_code: bool = True,
306
        is_sentence_transformer: bool = False,
307
        is_cross_encoder: bool = False,
308
        skip_tokenizer_init: bool = False,
309
        auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
        # Set this to avoid hanging issue
        default_torch_num_threads: Optional[int] = None,
    ) -> None:
        init_ctx = (nullcontext() if default_torch_num_threads is None else
                    set_default_torch_num_threads(default_torch_num_threads))

        with init_ctx:
            self._init(
                model_name=model_name,
                dtype=dtype,
                model_kwargs=model_kwargs,
                trust_remote_code=trust_remote_code,
                is_sentence_transformer=is_sentence_transformer,
                is_cross_encoder=is_cross_encoder,
                skip_tokenizer_init=skip_tokenizer_init,
                auto_cls=auto_cls,
            )

    def _init(
        self,
        model_name: str,
        dtype: str = "auto",
        *,
        model_kwargs: Optional[dict[str, Any]] = None,
        trust_remote_code: bool = True,
        is_sentence_transformer: bool = False,
        is_cross_encoder: bool = False,
        skip_tokenizer_init: bool = False,
        auto_cls: type[_BaseAutoModelClass] = AutoModelForCausalLM,
Woosuk Kwon's avatar
Woosuk Kwon committed
339
    ) -> None:
340
        model_name = maybe_model_redirect(model_name)
341
        self.model_name = model_name
342

343
344
        self.config = AutoConfig.from_pretrained(
            model_name,
345
            trust_remote_code=trust_remote_code,
346
347
        )
        self.device = self.get_default_device()
348
349
350
351
352
353
        self.dtype = torch_dtype = _get_and_verify_dtype(
            self.model_name,
            self.config,
            dtype=dtype,
            is_pooling_model=is_sentence_transformer or is_cross_encoder,
        )
354
355
356
357

        model_kwargs = model_kwargs if model_kwargs is not None else {}
        model_kwargs.setdefault("torch_dtype", torch_dtype)

358
        if is_sentence_transformer:
359
360
            # Lazy init required for AMD CI
            from sentence_transformers import SentenceTransformer
361
362
363
364
365

            self.model = SentenceTransformer(
                model_name,
                device=self.device,
                model_kwargs=model_kwargs,
366
                trust_remote_code=trust_remote_code,
367
            )
368
369
370
        elif is_cross_encoder:
            # Lazy init required for AMD CI
            from sentence_transformers import CrossEncoder
371
372
373
374
375

            self.model = CrossEncoder(
                model_name,
                device=self.device,
                automodel_args=model_kwargs,
376
                trust_remote_code=trust_remote_code,
377
            )
378
        else:
379
380
            model = auto_cls.from_pretrained(
                model_name,
381
                trust_remote_code=trust_remote_code,
382
383
384
                **model_kwargs,
            )

385
386
387
388
389
390
            # in case some unquantized custom models are not in same dtype
            if (getattr(model, "quantization_method", None) is None
                    and any(p.dtype != self.dtype
                            for p in model.parameters())):
                model = model.to(dtype=self.dtype)

391
392
393
            if (getattr(model, "quantization_method", None) != "bitsandbytes"
                    and len({p.device
                             for p in model.parameters()}) < 2):
394
                model = model.to(device=self.device)
395
396

            self.model = model
397

398
399
400
401
        if not skip_tokenizer_init:
            self.tokenizer = AutoTokenizer.from_pretrained(
                model_name,
                torch_dtype=torch_dtype,
402
                trust_remote_code=trust_remote_code,
403
            )
404

405
406
407
408
409
410
        # don't put this import at the top level
        # it will call torch.cuda.device_count()
        from transformers import AutoProcessor  # noqa: F401
        self.processor = AutoProcessor.from_pretrained(
            model_name,
            torch_dtype=torch_dtype,
411
            trust_remote_code=trust_remote_code,
412
        )
413
414
        if skip_tokenizer_init:
            self.tokenizer = self.processor.tokenizer
Woosuk Kwon's avatar
Woosuk Kwon committed
415

416
    def get_inputs(
Woosuk Kwon's avatar
Woosuk Kwon committed
417
        self,
418
        prompts: list[str],
419
        images: Optional[PromptImageInput] = None,
420
421
        videos: Optional[PromptVideoInput] = None,
        audios: Optional[PromptAudioInput] = None,
422
    ) -> list[Union[BatchFeature, BatchEncoding]]:
423
        if images is not None:
424
            assert len(prompts) == len(images)
425

426
427
428
429
430
431
        if videos is not None:
            assert len(prompts) == len(videos)

        if audios is not None:
            assert len(prompts) == len(audios)

432
        all_inputs: list[Union[BatchFeature, BatchEncoding]] = []
433
        for i, prompt in enumerate(prompts):
434
            processor_kwargs: dict[str, Any] = {
435
436
437
                "text": prompt,
                "return_tensors": "pt",
            }
Cyrus Leung's avatar
Cyrus Leung committed
438
439
440
441
            if images is not None and (image := images[i]) is not None:
                processor_kwargs["images"] = image
            if videos is not None and (video := videos[i]) is not None:
                processor_kwargs["videos"] = video
442
443
444
445
446
447
448
449
450
            if audios is not None and (audio_inputs := audios[i]) is not None:
                # HACK - not all processors take sampling_rate; we should
                # clean this up in the future.
                if len(audio_inputs) == 2:
                    audio, sr = audio_inputs
                    processor_kwargs["audio"] = audio
                    processor_kwargs["sampling_rate"] = sr
                else:
                    processor_kwargs["audio"] = audio_inputs
451
452

            inputs = self.processor(**processor_kwargs)
453
454
            if isinstance(inputs, BatchFeature):
                inputs = inputs.to(dtype=self.dtype)
455

456
457
458
459
            all_inputs.append(inputs)

        return all_inputs

460
461
462
463
464
465
466
467
468
    def get_prompt_embeddings(self, prompts: list[str]) -> list[torch.Tensor]:
        all_inputs = self.get_inputs(prompts)
        embeddings = []
        for inputs in all_inputs:
            input_ids = self.wrap_device(inputs)["input_ids"]
            embedding = self.model.get_input_embeddings()(input_ids).squeeze(0)
            embeddings.append(embedding)
        return embeddings

469
    def classify(self, prompts: list[str]) -> list[str]:
470
471
472
        # output is final logits
        all_inputs = self.get_inputs(prompts)
        outputs = []
473
474
        problem_type = getattr(self.config, "problem_type", "")

475
476
        for inputs in all_inputs:
            output = self.model(**self.wrap_device(inputs))
477
478
479
480
481
482
            if problem_type == "regression":
                logits = output.logits[0].tolist()
            elif problem_type == "multi_label_classification":
                logits = output.logits.sigmoid()[0].tolist()
            else:
                logits = output.logits.softmax(dim=-1)[0].tolist()
483
484
485
486
            outputs.append(logits)

        return outputs

487
488
    def generate(
        self,
489
        prompts: list[str],
490
        images: Optional[PromptImageInput] = None,
Cyrus Leung's avatar
Cyrus Leung committed
491
        videos: Optional[PromptVideoInput] = None,
492
493
        audios: Optional[PromptAudioInput] = None,
        **kwargs: Any,
494
    ) -> list[tuple[list[list[int]], list[str]]]:
495
496
497
498
499
        all_inputs = self.get_inputs(prompts,
                                     images=images,
                                     videos=videos,
                                     audios=audios)

500
        outputs: list[tuple[list[list[int]], list[str]]] = []
501
        for inputs in all_inputs:
Woosuk Kwon's avatar
Woosuk Kwon committed
502
            output_ids = self.model.generate(
503
                **self.wrap_device(inputs),
Woosuk Kwon's avatar
Woosuk Kwon committed
504
505
506
                use_cache=True,
                **kwargs,
            )
507
            output_str = self.processor.batch_decode(
Woosuk Kwon's avatar
Woosuk Kwon committed
508
509
510
                output_ids,
                skip_special_tokens=True,
                clean_up_tokenization_spaces=False,
511
512
            )
            output_ids = output_ids.cpu().tolist()
Woosuk Kwon's avatar
Woosuk Kwon committed
513
514
515
516
517
            outputs.append((output_ids, output_str))
        return outputs

    def generate_greedy(
        self,
518
        prompts: list[str],
Woosuk Kwon's avatar
Woosuk Kwon committed
519
        max_tokens: int,
520
        images: Optional[PromptImageInput] = None,
Cyrus Leung's avatar
Cyrus Leung committed
521
        videos: Optional[PromptVideoInput] = None,
522
        audios: Optional[PromptAudioInput] = None,
523
        **kwargs: Any,
524
    ) -> list[tuple[list[int], str]]:
525
526
        outputs = self.generate(prompts,
                                do_sample=False,
527
                                max_new_tokens=max_tokens,
Chang Su's avatar
Chang Su committed
528
                                images=images,
529
530
                                videos=videos,
                                audios=audios,
Chang Su's avatar
Chang Su committed
531
                                **kwargs)
532
533
534

        return [(output_ids[0], output_str[0])
                for output_ids, output_str in outputs]
535
536
537

    def generate_beam_search(
        self,
538
        prompts: list[str],
539
540
        beam_width: int,
        max_tokens: int,
541
542
543
        images: Optional[PromptImageInput] = None,
        videos: Optional[PromptVideoInput] = None,
        audios: Optional[PromptAudioInput] = None,
544
    ) -> list[tuple[list[list[int]], list[str]]]:
545
546
547
548
        outputs = self.generate(prompts,
                                do_sample=False,
                                max_new_tokens=max_tokens,
                                num_beams=beam_width,
549
550
551
552
553
                                num_return_sequences=beam_width,
                                images=images,
                                videos=videos,
                                audios=audios)

554
555
556
557
558
559
560
561
562
        for i in range(len(outputs)):
            output_ids, output_str = outputs[i]
            for j in range(len(output_ids)):
                output_ids[j] = [
                    x for x in output_ids[j]
                    if x != self.tokenizer.pad_token_id
                ]
            outputs[i] = (output_ids, output_str)
        return outputs
Woosuk Kwon's avatar
Woosuk Kwon committed
563

564
565
    def generate_greedy_logprobs(
        self,
566
        prompts: list[str],
567
        max_tokens: int,
568
        images: Optional[PromptImageInput] = None,
Cyrus Leung's avatar
Cyrus Leung committed
569
        videos: Optional[PromptVideoInput] = None,
570
        audios: Optional[PromptAudioInput] = None,
571
        **kwargs: Any,
572
    ) -> list[list[torch.Tensor]]:
573
574
575
576
        all_inputs = self.get_inputs(prompts,
                                     images=images,
                                     videos=videos,
                                     audios=audios)
577

578
        all_logprobs: list[list[torch.Tensor]] = []
579
        for inputs in all_inputs:
580
            output = self.model.generate(
581
                **self.wrap_device(inputs),
582
583
584
585
586
                use_cache=True,
                do_sample=False,
                max_new_tokens=max_tokens,
                output_hidden_states=True,
                return_dict_in_generate=True,
587
                **kwargs,
588
            )
589
590
            seq_logprobs = self._hidden_states_to_seq_logprobs(
                output.hidden_states)
591
592
593
            all_logprobs.append(seq_logprobs)
        return all_logprobs

594
    def _hidden_states_to_seq_logprobs(
595
        self,
596
597
        hidden_states: tuple[tuple[torch.Tensor, ...], ...],
    ) -> list[torch.Tensor]:
598
599
        output_embeddings = self.model.get_output_embeddings()

600
        seq_logprobs: list[torch.Tensor] = []
601
602
603
        for _, hidden_state in enumerate(hidden_states):
            last_hidden_states = hidden_state[-1][0]
            logits = torch.matmul(
604
605
606
607
                last_hidden_states.to(
                    device=output_embeddings.weight.device,
                    dtype=output_embeddings.weight.dtype,
                ),
608
                output_embeddings.weight.t(),
609
            )
610
611
            if getattr(output_embeddings, "bias", None) is not None:
                logits += output_embeddings.bias.unsqueeze(0)
612
613
614
            logprobs = F.log_softmax(logits, dim=-1, dtype=torch.float32)
            seq_logprobs.append(logprobs)

615
616
617
618
        return seq_logprobs

    def _hidden_states_to_logprobs(
        self,
619
        hidden_states: tuple[tuple[torch.Tensor, ...], ...],
620
        num_logprobs: Optional[int],
621
    ) -> tuple[list[dict[int, float]], int]:
622
623
624
        seq_logprobs = self._hidden_states_to_seq_logprobs(hidden_states)
        output_len = len(hidden_states)

625
        # convert to dict
626
        seq_logprobs_lst: list[dict[int, float]] = []
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
        for tok_idx, tok_logprobs in enumerate(seq_logprobs):
            # drop prompt logprobs
            if tok_idx == 0:
                tok_logprobs = tok_logprobs[-1, :].reshape(1, -1)
            topk = tok_logprobs.topk(num_logprobs)

            tok_logprobs_dct = {}
            for token_id, logprob in zip(topk.indices[0], topk.values[0]):
                tok_logprobs_dct[token_id.item()] = logprob.item()

            seq_logprobs_lst.append(tok_logprobs_dct)

        return (
            seq_logprobs_lst,
            output_len,
        )

644
645
    def generate_greedy_logprobs_limit(
        self,
646
        prompts: list[str],
647
        max_tokens: int,
648
        num_logprobs: Optional[int],
649
650
        images: Optional[PromptImageInput] = None,
        audios: Optional[PromptAudioInput] = None,
Cyrus Leung's avatar
Cyrus Leung committed
651
        videos: Optional[PromptVideoInput] = None,
652
        **kwargs: Any,
653
    ) -> list[TokensTextLogprobs]:
654
655
656
657
658
        all_inputs = self.get_inputs(prompts,
                                     images=images,
                                     videos=videos,
                                     audios=audios)

659
660
661
        all_logprobs: list[list[dict[int, float]]] = []
        all_output_ids: list[list[int]] = []
        all_output_strs: list[str] = []
662

663
        for inputs in all_inputs:
664
            output = self.model.generate(
665
                **self.wrap_device(inputs),
666
667
668
669
670
                use_cache=True,
                do_sample=False,
                max_new_tokens=max_tokens,
                output_hidden_states=True,
                return_dict_in_generate=True,
671
                **kwargs,
672
673
            )

674
675
676
677
678
679
680
681
682
683
684
685
            (
                seq_logprobs_lst,
                output_len,
            ) = self._hidden_states_to_logprobs(output.hidden_states,
                                                num_logprobs)

            all_logprobs.append(seq_logprobs_lst)
            seq_ids = output.sequences[0]
            output_len = len(seq_logprobs_lst)
            output_ids = seq_ids[-output_len:]
            all_output_ids.append(output_ids.tolist())
            all_output_strs.append(self.tokenizer.decode(output_ids))
686

687
688
689
690
        outputs = zip(all_output_ids, all_output_strs, all_logprobs)
        return [(output_ids, output_str, output_logprobs)
                for output_ids, output_str, output_logprobs in outputs]

691
692
693
    def encode(self, prompts: list[str], *args,
               **kwargs) -> list[list[torch.Tensor]]:
        return self.model.encode(prompts, *args, **kwargs)
694

695
696
697
698
699
700
    def predict(self, prompts: list[list[str]], *args,
                **kwargs) -> torch.Tensor:
        return self.model.predict(prompts,
                                  *args,
                                  convert_to_tensor=True,
                                  **kwargs)
701

702
703
704
705
    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
706
        del self.model
707
        cleanup_dist_env_and_memory()
708

Woosuk Kwon's avatar
Woosuk Kwon committed
709

Cyrus Leung's avatar
Cyrus Leung committed
710
@pytest.fixture(scope="session")
Woosuk Kwon's avatar
Woosuk Kwon committed
711
712
713
714
715
def hf_runner():
    return HfRunner


class VllmRunner:
716
717
    """
    The default value of some arguments have been modified from
718
    {class}`~vllm.LLM` as follows:
719

720
721
722
    - `trust_remote_code`: Set to `True` instead of `False` for convenience.
    - `seed`: Set to `0` instead of `None` for test reproducibility.
    - `max_model_len`: Set to `1024` instead of `None` to reduce memory usage.
723
724
    - `block_size`: To reduce memory usage, set default to `64` if on XPU
        devices, otherwise default to `16`.
725
726
    - `enable_chunked_prefill`: Set to `False` instead of `None` for
      test reproducibility.
727
    - `enforce_eager`: Set to `False` to test CUDA graph.
728
    """
Woosuk Kwon's avatar
Woosuk Kwon committed
729
730
731
732

    def __init__(
        self,
        model_name: str,
733
734
        runner: RunnerOption = "auto",
        convert: ConvertOption = "auto",
Woosuk Kwon's avatar
Woosuk Kwon committed
735
        tokenizer_name: Optional[str] = None,
736
        tokenizer_mode: str = "auto",
737
738
        trust_remote_code: bool = True,
        seed: Optional[int] = 0,
739
        max_model_len: Optional[int] = 1024,
740
        dtype: str = "auto",
741
        disable_log_stats: bool = True,
742
        tensor_parallel_size: int = 1,
743
        block_size: int = 16 if not torch.xpu.is_available() else 64,
744
        enable_chunked_prefill: Optional[bool] = False,
745
        swap_space: int = 4,
746
        enforce_eager: Optional[bool] = False,
747
748
        # Set this to avoid hanging issue
        default_torch_num_threads: Optional[int] = None,
749
        **kwargs,
Woosuk Kwon's avatar
Woosuk Kwon committed
750
    ) -> None:
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
        init_ctx = (nullcontext() if default_torch_num_threads is None else
                    set_default_torch_num_threads(default_torch_num_threads))

        with init_ctx:
            self.llm = LLM(
                model=model_name,
                runner=runner,
                convert=convert,
                tokenizer=tokenizer_name,
                tokenizer_mode=tokenizer_mode,
                trust_remote_code=trust_remote_code,
                dtype=dtype,
                seed=seed,
                swap_space=swap_space,
                enforce_eager=enforce_eager,
                disable_log_stats=disable_log_stats,
                tensor_parallel_size=tensor_parallel_size,
                max_model_len=max_model_len,
                block_size=block_size,
                enable_chunked_prefill=enable_chunked_prefill,
                **kwargs,
            )
Woosuk Kwon's avatar
Woosuk Kwon committed
773

774
    def get_inputs(
Woosuk Kwon's avatar
Woosuk Kwon committed
775
        self,
776
        prompts: Union[list[str], list[torch.Tensor], list[int]],
777
        images: Optional[PromptImageInput] = None,
778
779
        videos: Optional[PromptVideoInput] = None,
        audios: Optional[PromptAudioInput] = None,
780
    ) -> list[TextPrompt]:
781

782
783
784
785
786
        if any(x is not None and len(x) != len(prompts)
               for x in [images, videos, audios]):
            raise ValueError(
                "All non-None multimodal inputs must have the same length as "
                "prompts")
787

788
789
790
791
792
793
794
795
796
797
        inputs = []
        for i, prompt in enumerate(prompts):
            multi_modal_data = {}
            if images is not None and (image := images[i]) is not None:
                multi_modal_data["image"] = image
            if videos is not None and (video := videos[i]) is not None:
                multi_modal_data["video"] = video
            if audios is not None and (audio := audios[i]) is not None:
                multi_modal_data["audio"] = audio

798
            text_prompt_kwargs: dict[str, Any] = {
799
800
                "multi_modal_data": multi_modal_data or None
            }
801
802
803
804
805
806
807
            if isinstance(prompt, str):
                text_prompt_kwargs["prompt"] = prompt
            elif isinstance(prompt, list):
                text_prompt_kwargs["prompt_token_ids"] = prompt
            else:
                text_prompt_kwargs["prompt_embeds"] = prompt

808
            inputs.append(TextPrompt(**text_prompt_kwargs))
809
810
811
812
813

        return inputs

    def generate(
        self,
814
        prompts: Union[list[str], list[torch.Tensor]],
815
816
817
818
        sampling_params: SamplingParams,
        images: Optional[PromptImageInput] = None,
        videos: Optional[PromptVideoInput] = None,
        audios: Optional[PromptAudioInput] = None,
819
        **kwargs: Any,
820
    ) -> list[tuple[list[list[int]], list[str]]]:
821
822
823
824
825
        inputs = self.get_inputs(prompts,
                                 images=images,
                                 videos=videos,
                                 audios=audios)

826
827
828
        req_outputs = self.llm.generate(inputs,
                                        sampling_params=sampling_params,
                                        **kwargs)
829

830
        outputs: list[tuple[list[list[int]], list[str]]] = []
Woosuk Kwon's avatar
Woosuk Kwon committed
831
832
833
        for req_output in req_outputs:
            prompt_str = req_output.prompt
            prompt_ids = req_output.prompt_token_ids
834
835
            req_sample_output_ids: list[list[int]] = []
            req_sample_output_strs: list[str] = []
836
837
            for sample in req_output.outputs:
                output_str = sample.text
838
                output_ids = list(sample.token_ids)
839
                req_sample_output_ids.append(prompt_ids + output_ids)
840
                req_sample_output_strs.append((prompt_str or "") + output_str)
841
            outputs.append((req_sample_output_ids, req_sample_output_strs))
Woosuk Kwon's avatar
Woosuk Kwon committed
842
843
        return outputs

844
    @staticmethod
845
    def _final_steps_generate_w_logprobs(
846
847
848
        req_outputs: list[RequestOutput],
    ) -> list[TokensTextLogprobsPromptLogprobs]:
        outputs: list[TokensTextLogprobsPromptLogprobs] = []
849
        for req_output in req_outputs:
850
            assert len(req_output.outputs) > 0
851
852
            for sample in req_output.outputs:
                output_str = sample.text
853
                output_ids = list(sample.token_ids)
854
                output_logprobs = sample.logprobs
855
856
            outputs.append((output_ids, output_str, output_logprobs,
                            req_output.prompt_logprobs))
857
858
        return outputs

859
860
    def generate_w_logprobs(
        self,
861
        prompts: list[str],
862
        sampling_params: SamplingParams,
863
864
        images: Optional[PromptImageInput] = None,
        audios: Optional[PromptAudioInput] = None,
865
        videos: Optional[PromptVideoInput] = None,
866
        **kwargs: Any,
867
868
    ) -> Union[list[TokensTextLogprobs],
               list[TokensTextLogprobsPromptLogprobs]]:
869
870
871
872
        inputs = self.get_inputs(prompts,
                                 images=images,
                                 videos=videos,
                                 audios=audios)
873

874
875
876
        req_outputs = self.llm.generate(inputs,
                                        sampling_params=sampling_params,
                                        **kwargs)
877
878
879
880
881
882
883

        toks_str_logsprobs_prompt_logprobs = (
            self._final_steps_generate_w_logprobs(req_outputs))
        # Omit prompt logprobs if not required by sampling params
        return ([x[0:-1] for x in toks_str_logsprobs_prompt_logprobs]
                if sampling_params.prompt_logprobs is None else
                toks_str_logsprobs_prompt_logprobs)
884

Woosuk Kwon's avatar
Woosuk Kwon committed
885
886
    def generate_greedy(
        self,
887
        prompts: Union[list[str], list[torch.Tensor]],
Woosuk Kwon's avatar
Woosuk Kwon committed
888
        max_tokens: int,
889
        images: Optional[PromptImageInput] = None,
890
891
        videos: Optional[PromptVideoInput] = None,
        audios: Optional[PromptAudioInput] = None,
892
        **kwargs: Any,
893
    ) -> list[tuple[list[int], str]]:
Woosuk Kwon's avatar
Woosuk Kwon committed
894
        greedy_params = SamplingParams(temperature=0.0, max_tokens=max_tokens)
895
896
897
898
        outputs = self.generate(prompts,
                                greedy_params,
                                images=images,
                                videos=videos,
899
900
                                audios=audios,
                                **kwargs)
901
902
        return [(output_ids[0], output_str[0])
                for output_ids, output_str in outputs]
903

904
905
    def generate_greedy_logprobs(
        self,
906
        prompts: list[str],
907
        max_tokens: int,
908
        num_logprobs: Optional[int],
909
        num_prompt_logprobs: Optional[int] = None,
910
911
        images: Optional[PromptImageInput] = None,
        audios: Optional[PromptAudioInput] = None,
912
        videos: Optional[PromptVideoInput] = None,
913
914
        stop_token_ids: Optional[list[int]] = None,
        stop: Optional[list[str]] = None,
915
        **kwargs: Any,
916
917
    ) -> Union[list[TokensTextLogprobs],
               list[TokensTextLogprobsPromptLogprobs]]:
918
919
920
921
        greedy_logprobs_params = SamplingParams(
            temperature=0.0,
            max_tokens=max_tokens,
            logprobs=num_logprobs,
922
            prompt_logprobs=num_prompt_logprobs,
923
924
            stop_token_ids=stop_token_ids,
            stop=stop)
925
926
927
928
929

        return self.generate_w_logprobs(prompts,
                                        greedy_logprobs_params,
                                        images=images,
                                        audios=audios,
930
931
                                        videos=videos,
                                        **kwargs)
932

933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
    def generate_prompt_perplexity(self, prompts: list[str]) -> list[float]:
        """
        Return the perplexity score associated with generating the prompts

        :param prompts: list of prompts to score
        :return: perplexity score of each prompt
        """
        outputs = self.generate_greedy_logprobs(prompts,
                                                max_tokens=1,
                                                num_logprobs=None,
                                                num_prompt_logprobs=0)

        perplexities = []
        for output in outputs:
            output = cast(TokensTextLogprobsPromptLogprobs, output)
            token_datas = cast(list[Optional[dict[int, Logprob]]], output[3])
            assert token_datas[0] is None
            token_log_probs = []
            for token_data in token_datas[1:]:
                assert token_data is not None
                assert len(token_data) == 1
                token_log_prob = list(token_data.values())[0].logprob
                token_log_probs.append(token_log_prob)

            perplexity = math.exp(-sum(token_log_probs) / len(token_log_probs))
            perplexities.append(perplexity)

        return perplexities

962
    def generate_beam_search(
963
        self,
964
        prompts: list[str],
965
966
        beam_width: int,
        max_tokens: int,
967
968
969
        images: Optional[PromptImageInput] = None,
        videos: Optional[PromptVideoInput] = None,
        audios: Optional[PromptAudioInput] = None,
970
        concurrency_limit: Optional[int] = None,
971
    ) -> list[tuple[list[list[int]], list[str]]]:
972
973
974
975
976
        inputs = self.get_inputs(prompts,
                                 images=images,
                                 videos=videos,
                                 audios=audios)

977
978
979
980
        outputs = self.llm.beam_search(inputs,
                                       BeamSearchParams(beam_width=beam_width,
                                                        max_tokens=max_tokens),
                                       concurrency_limit=concurrency_limit)
981
982
983
984
985
986
987
        returned_outputs = []
        for output in outputs:
            token_ids = [x.tokens for x in output.sequences]
            texts = [x.text for x in output.sequences]
            returned_outputs.append((token_ids, texts))
        return returned_outputs

988
    def classify(self, prompts: list[str]) -> list[list[float]]:
989
        req_outputs = self.llm.classify(prompts)
990
991
        return [req_output.outputs.probs for req_output in req_outputs]

992
993
994
995
996
997
998
    def embed(self,
              prompts: list[str],
              images: Optional[PromptImageInput] = None,
              videos: Optional[PromptVideoInput] = None,
              audios: Optional[PromptAudioInput] = None,
              *args,
              **kwargs) -> list[list[float]]:
Cyrus Leung's avatar
Cyrus Leung committed
999
1000
1001
1002
1003
        inputs = self.get_inputs(prompts,
                                 images=images,
                                 videos=videos,
                                 audios=audios)

1004
        req_outputs = self.llm.embed(inputs, *args, **kwargs)
Cyrus Leung's avatar
Cyrus Leung committed
1005
        return [req_output.outputs.embedding for req_output in req_outputs]
1006

1007
    def encode(self, prompts: list[str]) -> list[list[float]]:
1008
        req_outputs = self.llm.encode(prompts)
1009
1010
        return [req_output.outputs.data for req_output in req_outputs]

1011
1012
1013
1014
    def reward(self, prompts: list[str]) -> list[list[float]]:
        req_outputs = self.llm.reward(prompts)
        return [req_output.outputs.data for req_output in req_outputs]

1015
1016
    def score(
        self,
1017
1018
        text_1: Union[str, list[str]],
        text_2: Union[str, list[str]],
1019
1020
        *args,
        **kwargs,
1021
    ) -> list[float]:
1022
        req_outputs = self.llm.score(text_1, text_2, *args, **kwargs)
1023
        return [req_output.outputs.score for req_output in req_outputs]
1024

1025
    def apply_model(self, func: Callable[[nn.Module], _R]) -> list[_R]:
1026
        return self.llm.apply_model(func)
1027

1028
1029
1030
    def get_llm(self) -> LLM:
        return self.llm

1031
1032
1033
1034
    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_value, traceback):
1035
        del self.llm
1036
        cleanup_dist_env_and_memory()
1037

Woosuk Kwon's avatar
Woosuk Kwon committed
1038

1039
@pytest.fixture(scope="session")
Woosuk Kwon's avatar
Woosuk Kwon committed
1040
1041
def vllm_runner():
    return VllmRunner
1042
1043


1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
@pytest.fixture()
def temporary_enable_log_propagate():
    import logging
    logger = logging.getLogger("vllm")
    logger.propagate = True
    yield
    logger.propagate = False


@pytest.fixture()
def caplog_vllm(temporary_enable_log_propagate, caplog):
    # To capture vllm log, we should enable propagate=True temporarily
    # because caplog depends on logs propagated to the root logger.
    yield caplog
1058
1059
1060
1061
1062
1063
1064


@pytest.fixture(scope="session")
def num_gpus_available():
    """Get number of GPUs without initializing the CUDA context
    in current process."""

1065
1066
    from vllm.platforms import current_platform
    return current_platform.device_count()
1067
1068
1069


temp_dir = tempfile.gettempdir()
1070
1071
_dummy_opt_path = os.path.join(temp_dir, "dummy_opt")
_dummy_llava_path = os.path.join(temp_dir, "dummy_llava")
1072
_dummy_gemma2_embedding_path = os.path.join(temp_dir, "dummy_gemma2_embedding")
1073
1074
1075
1076


@pytest.fixture
def dummy_opt_path():
1077
1078
    json_path = os.path.join(_dummy_opt_path, "config.json")
    if not os.path.exists(_dummy_opt_path):
1079
        snapshot_download(repo_id="facebook/opt-125m",
1080
                          local_dir=_dummy_opt_path,
1081
1082
1083
1084
1085
                          ignore_patterns=[
                              "*.bin", "*.bin.index.json", "*.pt", "*.h5",
                              "*.msgpack"
                          ])
        assert os.path.exists(json_path)
1086
        with open(json_path) as f:
1087
1088
1089
1090
            config = json.load(f)
        config["architectures"] = ["MyOPTForCausalLM"]
        with open(json_path, "w") as f:
            json.dump(config, f)
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
    return _dummy_opt_path


@pytest.fixture
def dummy_llava_path():
    json_path = os.path.join(_dummy_llava_path, "config.json")
    if not os.path.exists(_dummy_llava_path):
        snapshot_download(repo_id="llava-hf/llava-1.5-7b-hf",
                          local_dir=_dummy_llava_path,
                          ignore_patterns=[
                              "*.bin", "*.bin.index.json", "*.pt", "*.h5",
                              "*.msgpack"
                          ])
        assert os.path.exists(json_path)
1105
        with open(json_path) as f:
1106
1107
1108
1109
1110
            config = json.load(f)
        config["architectures"] = ["MyLlava"]
        with open(json_path, "w") as f:
            json.dump(config, f)
    return _dummy_llava_path
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123


@pytest.fixture
def dummy_gemma2_embedding_path():
    json_path = os.path.join(_dummy_gemma2_embedding_path, "config.json")
    if not os.path.exists(_dummy_gemma2_embedding_path):
        snapshot_download(repo_id="BAAI/bge-multilingual-gemma2",
                          local_dir=_dummy_gemma2_embedding_path,
                          ignore_patterns=[
                              "*.bin", "*.bin.index.json", "*.pt", "*.h5",
                              "*.msgpack"
                          ])
        assert os.path.exists(json_path)
1124
        with open(json_path) as f:
1125
1126
1127
1128
1129
            config = json.load(f)
        config["architectures"] = ["MyGemma2Embedding"]
        with open(json_path, "w") as f:
            json.dump(config, f)
    return _dummy_gemma2_embedding_path
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147


# Add the flag `--optional` to allow run tests
# that are marked with @pytest.mark.optional
def pytest_addoption(parser):
    parser.addoption("--optional",
                     action="store_true",
                     default=False,
                     help="run optional test")


def pytest_collection_modifyitems(config, items):
    if config.getoption("--optional"):
        # --optional given in cli: do not skip optional tests
        return
    skip_optional = pytest.mark.skip(reason="need --optional option to run")
    for item in items:
        if "optional" in item.keywords:
1148
            item.add_marker(skip_optional)
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160


@pytest.fixture(scope="session")
def cli_config_file():
    """Return the path to the CLI config file."""
    return os.path.join(_TEST_DIR, "config", "test_config.yaml")


@pytest.fixture(scope="session")
def cli_config_file_with_model():
    """Return the path to the CLI config file with model."""
    return os.path.join(_TEST_DIR, "config", "test_config_with_model.yaml")
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276


class AssetHandler(http.server.BaseHTTPRequestHandler):
    # _IMAGE_CACHE : Dict[str, bytes] = {}

    def log_message(self, *args, **kwargs):
        pass

    def do_GET(self):
        # Accepts paths like: /1280px-Venn_diagram_rgb.jpg
        filename = self.path.lstrip("/")
        if not filename or "." not in filename:
            self.send_error(404, "Missing filename (expected /<name>.<ext>)")
            return

        base, ext = filename.rsplit(".", 1)
        ext = ext.lower()

        if ext not in ["jpg", "png"]:
            self.send_error(404, f"Unsupported extension: .{ext}")
            return

        try:
            data = ImageAsset(base).read_bytes(ext=ext)
        except Exception as e:
            self.send_error(500, f"Failed to load asset: {ext} {base} {e} ")
            return

        ctype, _ = mimetypes.guess_type(filename)
        if ctype is None:
            ctype = {"jpg": "image/jpg", "png": "image/png"}[ext]
        self.send_response(200)
        self.send_header("Content-Type", ctype)
        self.send_header("Content-Length", str(len(data)))
        self.end_headers()
        self.wfile.write(data)


def _find_free_port() -> int:
    with socket.socket() as s:
        s.bind(("127.0.0.1", 0))
        return s.getsockname()[1]


class LocalAssetServer:

    address: str
    port: int
    server: Optional[http.server.ThreadingHTTPServer]
    thread: Optional[threading.Thread]

    def __init__(self, address: str = "127.0.0.1") -> None:
        self.address = address
        self.port = -1
        self.server = None
        self.thread = None

    def __enter__(self):
        self.port = _find_free_port()
        self.server = http.server.ThreadingHTTPServer(
            (self.address, self.port), AssetHandler)
        self.thread = threading.Thread(target=self.server.serve_forever,
                                       daemon=True)
        self.thread.start()
        return self

    def __exit__(self, exc_type, exc_value, traceback):
        if self.server:
            self.server.shutdown()
            del self.server

        if self.thread:
            self.thread.join()
            del self.thread

        if exc_type is None:
            return None

        return False

    @property
    def base_url(self) -> str:
        assert self.port is not None
        return f"http://{self.address}:{self.port}"

    def url_for(self, name: str) -> str:
        """e.g., name='RGBA_comp.png' -> 'http://127.0.0.1:PORT/RGBA_comp.png'"""
        return f"{self.base_url}/{name}"

    def get_image_asset(self, name: str) -> Image.Image:
        return fetch_image(self.url_for(name))


@pytest.fixture(scope="session")
def local_asset_server() -> Generator[LocalAssetServer, None, None]:
    """
    Starts a thread based HTTP server bound to 127.0.0.1 on a random free port. 
    The server currently servers images at:
    http://127.0.0.1:<port>/<name>.<ext>
    """
    with LocalAssetServer() as srv:
        yield srv


@pytest.fixture
def image_url(request, local_asset_server) -> str:
    # request.param is one of the IMAGE_ASSETS filenames
    name = request.param
    return local_asset_server.url_for(name)


@pytest.fixture
def image_urls(request, local_asset_server) -> list[str]:
    """Indirect fixture: takes a list of names, returns list of full URLs."""
    names: list[str] = request.param
    return [local_asset_server.url_for(name) for name in names]