utils.py 12.6 KB
Newer Older
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
import argparse
4
import contextlib
5
import multiprocessing
6
import time
7
import weakref
8
from collections.abc import Sequence
9
from contextlib import AbstractContextManager
10
from multiprocessing import connection
11
from multiprocessing.process import BaseProcess
12
13
14
15
16
17
18
19
20
21
from typing import (
    TYPE_CHECKING,
    Any,
    Callable,
    Generic,
    Optional,
    TypeVar,
    Union,
    overload,
)
22
23

import torch
24
from torch.autograd.profiler import record_function
25

26
import vllm.envs as envs
27
from vllm.logger import init_logger
28
29
30
31
32
33
34
from vllm.usage.usage_lib import UsageContext, is_usage_stats_enabled, usage_message
from vllm.utils import (
    get_open_port,
    get_open_zmq_ipc_path,
    get_tcp_uri,
    kill_process_tree,
)
35

36
if TYPE_CHECKING:
37
38
    import numpy as np

39
    from vllm.v1.engine.coordinator import DPCoordinator
40
    from vllm.v1.engine.utils import CoreEngineActorManager, CoreEngineProcManager
41

42
logger = init_logger(__name__)
43
44
45
46

T = TypeVar("T")


47
class ConstantList(Generic[T], Sequence):
48
    def __init__(self, x: list[T]) -> None:
49
50
51
        self._x = x

    def append(self, item):
52
        raise TypeError("Cannot append to a constant list")
53
54

    def extend(self, item):
55
        raise TypeError("Cannot extend a constant list")
56
57

    def insert(self, item):
58
        raise TypeError("Cannot insert into a constant list")
59
60

    def pop(self, item):
61
        raise TypeError("Cannot pop from a constant list")
62
63

    def remove(self, item):
64
        raise TypeError("Cannot remove from a constant list")
65
66

    def clear(self):
67
        raise TypeError("Cannot clear a constant list")
68

69
70
    def index(self, item: T, start: int = 0, stop: Optional[int] = None) -> int:
        return self._x.index(item, start, stop if stop is not None else len(self._x))
71
72

    @overload
73
    def __getitem__(self, item: int) -> T: ...
74
75

    @overload
76
    def __getitem__(self, s: slice, /) -> list[T]: ...
77

78
    def __getitem__(self, item: Union[int, slice]) -> Union[T, list[T]]:
79
80
81
        return self._x[item]

    @overload
82
    def __setitem__(self, item: int, value: T): ...
83
84

    @overload
85
    def __setitem__(self, s: slice, value: T, /): ...
86

87
    def __setitem__(self, item: Union[int, slice], value: Union[T, list[T]]):
88
        raise TypeError("Cannot set item in a constant list")
89
90

    def __delitem__(self, item):
91
        raise TypeError("Cannot delete item from a constant list")
92
93
94
95
96
97
98
99
100

    def __iter__(self):
        return iter(self._x)

    def __contains__(self, item):
        return item in self._x

    def __len__(self):
        return len(self._x)
101

102
103
104
    def __repr__(self):
        return f"ConstantList({self._x})"

105

106
class CpuGpuBuffer:
107
    """Buffer to easily copy tensors between CPU and GPU."""
108
109
110

    def __init__(
        self,
111
        *size: Union[int, torch.SymInt],
112
113
114
        dtype: torch.dtype,
        device: torch.device,
        pin_memory: bool,
115
116
        with_numpy: bool = True,
    ) -> None:
117
        self.cpu = torch.zeros(*size, dtype=dtype, device="cpu", pin_memory=pin_memory)
118
        self.gpu = torch.zeros_like(self.cpu, device=device)
119
120
121
122
123
124
125
126
        self.np: np.ndarray
        # To keep type hints simple (avoiding generics and subclasses), we
        # only conditionally create the numpy array attribute. This can cause
        # AttributeError if `self.np` is accessed when `with_numpy=False`.
        if with_numpy:
            if dtype == torch.bfloat16:
                raise ValueError(
                    "Bfloat16 torch tensors cannot be directly cast to a "
127
128
                    "numpy array, so call CpuGpuBuffer with with_numpy=False"
                )
129
            self.np = self.cpu.numpy()
130
131
132
133
134
135
136
137
138
139
140
141
142
143

    def copy_to_gpu(self, n: Optional[int] = None) -> torch.Tensor:
        if n is None:
            return self.gpu.copy_(self.cpu, non_blocking=True)
        return self.gpu[:n].copy_(self.cpu[:n], non_blocking=True)

    def copy_to_cpu(self, n: Optional[int] = None) -> torch.Tensor:
        """NOTE: Because this method is non-blocking, explicit synchronization
        is needed to ensure the data is copied to CPU."""
        if n is None:
            return self.cpu.copy_(self.gpu, non_blocking=True)
        return self.cpu[:n].copy_(self.gpu[:n], non_blocking=True)


144
def get_engine_client_zmq_addr(local_only: bool, host: str, port: int = 0) -> str:
145
    """Assign a new ZMQ socket address.
Rui Qiao's avatar
Rui Qiao committed
146

147
148
    If local_only is True, participants are colocated and so a unique IPC
    address will be returned.
Rui Qiao's avatar
Rui Qiao committed
149

150
151
    Otherwise, the provided host and port will be used to construct a TCP
    address (port == 0 means assign an available port)."""
Rui Qiao's avatar
Rui Qiao committed
152

153
154
155
156
157
    return (
        get_open_zmq_ipc_path()
        if local_only
        else (get_tcp_uri(host, port or get_open_port()))
    )
Rui Qiao's avatar
Rui Qiao committed
158
159


160
161
class APIServerProcessManager:
    """Manages a group of API server processes.
162

163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
    Handles creation, monitoring, and termination of API server worker
    processes. Also monitors extra processes to check if they are healthy.
    """

    def __init__(
        self,
        target_server_fn: Callable,
        listen_address: str,
        sock: Any,
        args: argparse.Namespace,
        num_servers: int,
        input_addresses: list[str],
        output_addresses: list[str],
        stats_update_address: Optional[str] = None,
    ):
        """Initialize and start API server worker processes.
179

180
181
182
183
184
185
186
187
        Args:
            target_server_fn: Function to call for each API server process
            listen_address: Address to listen for client connections
            sock: Socket for client connections
            args: Command line arguments
            num_servers: Number of API server processes to start
            input_addresses: Input addresses for each API server
            output_addresses: Output addresses for each API server
188
            stats_update_address: Optional stats update address
189
190
191
192
        """
        self.listen_address = listen_address
        self.sock = sock
        self.args = args
193

194
195
196
197
        # Start API servers
        spawn_context = multiprocessing.get_context("spawn")
        self.processes: list[BaseProcess] = []

198
199
200
        for i, in_addr, out_addr in zip(
            range(num_servers), input_addresses, output_addresses
        ):
201
202
203
            client_config = {
                "input_address": in_addr,
                "output_address": out_addr,
204
                "client_count": num_servers,
205
                "client_index": i,
206
207
208
209
            }
            if stats_update_address is not None:
                client_config["stats_update_address"] = stats_update_address

210
211
212
213
214
            proc = spawn_context.Process(
                target=target_server_fn,
                name=f"ApiServer_{i}",
                args=(listen_address, sock, args, client_config),
            )
215
216
217
218
219
220
221
222
223
224
225
226
227
228
            self.processes.append(proc)
            proc.start()

        logger.info("Started %d API server processes", len(self.processes))

        # Shutdown only the API server processes on garbage collection
        # The extra processes are managed by their owners
        self._finalizer = weakref.finalize(self, shutdown, self.processes)

    def close(self) -> None:
        self._finalizer()


def wait_for_completion_or_failure(
229
230
231
232
233
234
    api_server_manager: APIServerProcessManager,
    engine_manager: Optional[
        Union["CoreEngineProcManager", "CoreEngineActorManager"]
    ] = None,
    coordinator: Optional["DPCoordinator"] = None,
) -> None:
235
    """Wait for all processes to complete or detect if any fail.
236

237
    Raises an exception if any process exits with a non-zero status.
Rui Qiao's avatar
Rui Qiao committed
238
239
240
241
242
243
244

    Args:
        api_server_manager: The manager for API servers.
        engine_manager: The manager for engine processes.
            If CoreEngineProcManager, it manages local engines;
            if CoreEngineActorManager, it manages all engines.
        coordinator: The coordinator for data parallel.
245
246
    """

247
    from vllm.v1.engine.utils import CoreEngineActorManager, CoreEngineProcManager
248

249
250
251
252
253
    try:
        logger.info("Waiting for API servers to complete ...")
        # Create a mapping of sentinels to their corresponding processes
        # for efficient lookup
        sentinel_to_proc: dict[Any, BaseProcess] = {
254
            proc.sentinel: proc for proc in api_server_manager.processes
255
256
257
258
259
        }

        if coordinator:
            sentinel_to_proc[coordinator.proc.sentinel] = coordinator.proc

Rui Qiao's avatar
Rui Qiao committed
260
261
262
        actor_run_refs = []
        if isinstance(engine_manager, CoreEngineProcManager):
            for proc in engine_manager.processes:
263
                sentinel_to_proc[proc.sentinel] = proc
Rui Qiao's avatar
Rui Qiao committed
264
265
        elif isinstance(engine_manager, CoreEngineActorManager):
            actor_run_refs = engine_manager.get_run_refs()
266
267

        # Check if any process terminates
Rui Qiao's avatar
Rui Qiao committed
268
        while sentinel_to_proc or actor_run_refs:
269
            # Wait for any process to terminate
270
            ready_sentinels: list[Any] = connection.wait(sentinel_to_proc, timeout=5)
271
272
273
274
275
276
277
278
279

            # Process any terminated processes
            for sentinel in ready_sentinels:
                proc = sentinel_to_proc.pop(sentinel)

                # Check if process exited with error
                if proc.exitcode != 0:
                    raise RuntimeError(
                        f"Process {proc.name} (PID: {proc.pid}) "
280
281
                        f"died with exit code {proc.exitcode}"
                    )
Rui Qiao's avatar
Rui Qiao committed
282
283
284

            if actor_run_refs:
                import ray
285

Rui Qiao's avatar
Rui Qiao committed
286
287
                _, actor_run_refs = ray.wait(actor_run_refs, timeout=5)

288
289
290
    except KeyboardInterrupt:
        logger.info("Received KeyboardInterrupt, shutting down API servers...")
    except Exception as e:
291
        logger.exception("Exception occurred while running API servers: %s", str(e))
292
293
294
295
296
297
        raise
    finally:
        logger.info("Terminating remaining processes ...")
        api_server_manager.close()
        if coordinator:
            coordinator.close()
Rui Qiao's avatar
Rui Qiao committed
298
299
        if engine_manager:
            engine_manager.close()
300
301


Robert Shaw's avatar
Robert Shaw committed
302
# Note(rob): shutdown function cannot be a bound method,
303
304
# else the gc cannot collect the object.
def shutdown(procs: list[BaseProcess]):
Robert Shaw's avatar
Robert Shaw committed
305
    # Shutdown the process.
306
307
308
309
310
311
312
313
314
315
316
317
318
319
    for proc in procs:
        if proc.is_alive():
            proc.terminate()

    # Allow 5 seconds for remaining procs to terminate.
    deadline = time.monotonic() + 5
    for proc in procs:
        remaining = deadline - time.monotonic()
        if remaining <= 0:
            break
        if proc.is_alive():
            proc.join(remaining)

    for proc in procs:
320
321
        if proc.is_alive() and (pid := proc.pid) is not None:
            kill_process_tree(pid)
Robert Shaw's avatar
Robert Shaw committed
322

323

324
325
326
def copy_slice(
    from_tensor: torch.Tensor, to_tensor: torch.Tensor, length: int
) -> torch.Tensor:
327
328
329
330
331
    """
    Copy the first length elements of a tensor into another tensor in a
    non-blocking manner.

    Used to copy pinned CPU tensor data to pre-allocated GPU tensors.
332
333

    Returns the sliced target tensor.
334
    """
335
    return to_tensor[:length].copy_(from_tensor[:length], non_blocking=True)
336
337


338
def report_usage_stats(
339
340
    vllm_config, usage_context: UsageContext = UsageContext.ENGINE_CONTEXT
) -> None:
341
342
343
344
345
346
347
    """Report usage statistics if enabled."""

    if not is_usage_stats_enabled():
        return

    from vllm.model_executor.model_loader import get_architecture_class_name

348
349
    parallel_config = vllm_config.parallel_config

350
351
352
353
354
    usage_message.report_usage(
        get_architecture_class_name(vllm_config.model_config),
        usage_context,
        extra_kvs={
            # Common configuration
355
            "dtype": str(vllm_config.model_config.dtype),
356
            "tensor_parallel_size": parallel_config.tensor_parallel_size,
357
358
359
            "block_size": vllm_config.cache_config.block_size,
            "gpu_memory_utilization": vllm_config.cache_config.gpu_memory_utilization,
            "kv_cache_memory_bytes": vllm_config.cache_config.kv_cache_memory_bytes,
360
            # Quantization
361
362
            "quantization": vllm_config.model_config.quantization,
            "kv_cache_dtype": str(vllm_config.cache_config.cache_dtype),
363
            # Feature flags
364
365
366
            "enable_lora": bool(vllm_config.lora_config),
            "enable_prefix_caching": vllm_config.cache_config.enable_prefix_caching,
            "enforce_eager": vllm_config.model_config.enforce_eager,
367
            "disable_custom_all_reduce": parallel_config.disable_custom_all_reduce,
368
369
        },
    )
370
371


372
373
374
_PROFILER_FUNC = None


375
def record_function_or_nullcontext(name: str) -> AbstractContextManager:
376
377
378
379
380
381
382
    global _PROFILER_FUNC

    # fast path assume it is set
    if _PROFILER_FUNC is not None:
        return _PROFILER_FUNC(name)

    func = contextlib.nullcontext
383
    if envs.VLLM_CUSTOM_SCOPES_FOR_PROFILING:
384
385
386
        func = record_function
    elif envs.VLLM_NVTX_SCOPES_FOR_PROFILING:
        import nvtx
387

388
389
390
391
        func = nvtx.annotate

    _PROFILER_FUNC = func
    return func(name)