sequence.py 17.5 KB
Newer Older
1
"""Sequence and its related classes."""
2
import copy
Woosuk Kwon's avatar
Woosuk Kwon committed
3
import enum
4
from dataclasses import dataclass
Zhuohan Li's avatar
Zhuohan Li committed
5
from typing import Dict, List, Optional, Union
Woosuk Kwon's avatar
Woosuk Kwon committed
6

Woosuk Kwon's avatar
Woosuk Kwon committed
7
8
from vllm.block import LogicalTokenBlock
from vllm.sampling_params import SamplingParams
9
from vllm.lora.request import LoRARequest
Woosuk Kwon's avatar
Woosuk Kwon committed
10

11
12
13
PromptLogprobs = List[Optional[Dict[int, float]]]
SampleLogprobs = List[Dict[int, float]]

Woosuk Kwon's avatar
Woosuk Kwon committed
14
15

class SequenceStatus(enum.Enum):
16
    """Status of a sequence."""
17
    WAITING = enum.auto()
Woosuk Kwon's avatar
Woosuk Kwon committed
18
    RUNNING = enum.auto()
Woosuk Kwon's avatar
Woosuk Kwon committed
19
    SWAPPED = enum.auto()
Zhuohan Li's avatar
Zhuohan Li committed
20
21
    FINISHED_STOPPED = enum.auto()
    FINISHED_LENGTH_CAPPED = enum.auto()
22
    FINISHED_ABORTED = enum.auto()
Lily Liu's avatar
Lily Liu committed
23
    FINISHED_IGNORED = enum.auto()
Zhuohan Li's avatar
Zhuohan Li committed
24
25
26
27
28
29

    @staticmethod
    def is_finished(status: "SequenceStatus") -> bool:
        return status in [
            SequenceStatus.FINISHED_STOPPED,
            SequenceStatus.FINISHED_LENGTH_CAPPED,
30
            SequenceStatus.FINISHED_ABORTED,
31
            SequenceStatus.FINISHED_IGNORED,
Zhuohan Li's avatar
Zhuohan Li committed
32
33
34
35
36
37
38
39
        ]

    @staticmethod
    def get_finished_reason(status: "SequenceStatus") -> Union[str, None]:
        if status == SequenceStatus.FINISHED_STOPPED:
            finish_reason = "stop"
        elif status == SequenceStatus.FINISHED_LENGTH_CAPPED:
            finish_reason = "length"
40
41
        elif status == SequenceStatus.FINISHED_ABORTED:
            finish_reason = "abort"
Lily Liu's avatar
Lily Liu committed
42
        elif status == SequenceStatus.FINISHED_IGNORED:
43
44
45
            # The ignored sequences are the sequences whose prompt lengths
            # are longer than the model's length cap. Therefore, the stop
            # reason should also be "length" as in OpenAI API.
Lily Liu's avatar
Lily Liu committed
46
            finish_reason = "length"
Zhuohan Li's avatar
Zhuohan Li committed
47
48
49
        else:
            finish_reason = None
        return finish_reason
Woosuk Kwon's avatar
Woosuk Kwon committed
50

51

52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
@dataclass
class RequestMetrics:
    """Metrics associated with a request.

    Args:
        arrival_time: The time when the request arrived.
        first_scheduled_time: The time when the request was first scheduled.
        first_token_time: The time when the first token was generated.
        time_in_queue: The time the request spent in the queue.
        finished_time: The time when the request was finished.
    """
    arrival_time: float
    last_token_time: float
    first_scheduled_time: Optional[float]
    first_token_time: Optional[float]
    time_in_queue: Optional[float]
    finished_time: Optional[float] = None


71
class SequenceData:
72
73
74
75
76
77
78
79
80
81
    """Data associated with a sequence.

    Args:
        prompt_token_ids: The token IDs of the prompt.

    Attributes:
        prompt_token_ids: The token IDs of the prompt.
        output_token_ids: The token IDs of the output.
        cumulative_logprob: The cumulative log probability of the output.
    """
82
83
84
85
86
87
88

    def __init__(
        self,
        prompt_token_ids: List[int],
    ) -> None:
        self.prompt_token_ids = prompt_token_ids
        self.output_token_ids: List[int] = []
89
90
        self.cumulative_logprob = 0.0

91
    def append_token_id(self, token_id: int, logprob: float) -> None:
92
93
        self.output_token_ids.append(token_id)
        self.cumulative_logprob += logprob
94
95
96
97

    def get_len(self) -> int:
        return len(self.output_token_ids) + len(self.prompt_token_ids)

98
99
100
    def get_prompt_len(self) -> int:
        return len(self.prompt_token_ids)

101
102
103
    def get_output_len(self) -> int:
        return len(self.output_token_ids)

104
105
106
107
108
109
110
111
112
113
114
    def get_token_ids(self) -> List[int]:
        return self.prompt_token_ids + self.output_token_ids

    def get_last_token_id(self) -> int:
        if not self.output_token_ids:
            return self.prompt_token_ids[-1]
        return self.output_token_ids[-1]

    def __repr__(self) -> str:
        return (f"SequenceData("
                f"prompt_token_ids={self.prompt_token_ids}, "
115
116
                f"output_token_ids={self.output_token_ids}, "
                f"cumulative_logprob={self.cumulative_logprob})")
117
118


Woosuk Kwon's avatar
Woosuk Kwon committed
119
class Sequence:
120
121
122
123
124
125
126
127
    """Stores the data, status, and block information of a sequence.

    Args:
        seq_id: The ID of the sequence.
        prompt: The prompt of the sequence.
        prompt_token_ids: The token IDs of the prompt.
        block_size: The block size of the sequence. Should be the same as the
            block size used by the block manager and cache engine.
128
        lora_request: LoRA request.
129
    """
Woosuk Kwon's avatar
Woosuk Kwon committed
130
131
132
133

    def __init__(
        self,
        seq_id: int,
134
        prompt: str,
135
        prompt_token_ids: List[int],
Woosuk Kwon's avatar
Woosuk Kwon committed
136
        block_size: int,
137
        lora_request: Optional[LoRARequest] = None,
Woosuk Kwon's avatar
Woosuk Kwon committed
138
139
    ) -> None:
        self.seq_id = seq_id
140
        self.prompt = prompt
Woosuk Kwon's avatar
Woosuk Kwon committed
141
        self.block_size = block_size
142
        self.lora_request = lora_request
Woosuk Kwon's avatar
Woosuk Kwon committed
143

144
        self.data = SequenceData(prompt_token_ids)
145
        self.output_logprobs: SampleLogprobs = []
146
        self.output_text = ""
147

Woosuk Kwon's avatar
Woosuk Kwon committed
148
        self.logical_token_blocks: List[LogicalTokenBlock] = []
149
        # Initialize the logical token blocks with the prompt token ids.
150
        self._append_tokens_to_blocks(prompt_token_ids)
151
        self.status = SequenceStatus.WAITING
Woosuk Kwon's avatar
Woosuk Kwon committed
152

153
154
155
156
157
158
        # Used for incremental detokenization
        self.prefix_offset = 0
        self.read_offset = 0
        # Input + output tokens
        self.tokens: Optional[List[str]] = None

159
160
161
162
    @property
    def lora_int_id(self) -> int:
        return self.lora_request.lora_int_id if self.lora_request else 0

163
164
    def hash_of_block(self, logical_idx: int) -> int:
        # Compute the number of tokens in the sequence
165
166
        # TODO: The current hashing function is O(L^2). We should optimize
        # this in the future.
167
168
169
170
171
172
        num_tokens = self.num_hashed_tokens_of_block(logical_idx)
        return hash(tuple(self.data.get_token_ids()[0:num_tokens]))

    def num_hashed_tokens_of_block(self, logical_idx: int):
        return logical_idx * self.block_size + self.block_size

173
    def _append_logical_block(self) -> None:
Woosuk Kwon's avatar
Woosuk Kwon committed
174
175
176
177
178
179
        block = LogicalTokenBlock(
            block_number=len(self.logical_token_blocks),
            block_size=self.block_size,
        )
        self.logical_token_blocks.append(block)

180
    def _append_tokens_to_blocks(self, token_ids: List[int]) -> None:
181
182
        cursor = 0
        while cursor < len(token_ids):
Woosuk Kwon's avatar
Woosuk Kwon committed
183
            if not self.logical_token_blocks:
184
                self._append_logical_block()
Woosuk Kwon's avatar
Woosuk Kwon committed
185
186
187

            last_block = self.logical_token_blocks[-1]
            if last_block.is_full():
188
                self._append_logical_block()
Woosuk Kwon's avatar
Woosuk Kwon committed
189
190
191
                last_block = self.logical_token_blocks[-1]

            num_empty_slots = last_block.get_num_empty_slots()
192
193
194
            last_block.append_tokens(token_ids[cursor:cursor +
                                               num_empty_slots])
            cursor += num_empty_slots
Woosuk Kwon's avatar
Woosuk Kwon committed
195

196
197
198
199
200
    def append_token_id(
        self,
        token_id: int,
        logprobs: Dict[int, float],
    ) -> None:
201
        assert token_id in logprobs
202
        self._append_tokens_to_blocks([token_id])
203
        self.output_logprobs.append(logprobs)
204
        self.data.append_token_id(token_id, logprobs[token_id])
205

Woosuk Kwon's avatar
Woosuk Kwon committed
206
    def get_len(self) -> int:
207
        return self.data.get_len()
Woosuk Kwon's avatar
Woosuk Kwon committed
208

209
210
211
    def get_prompt_len(self) -> int:
        return self.data.get_prompt_len()

212
213
214
    def get_output_len(self) -> int:
        return self.data.get_output_len()

Woosuk Kwon's avatar
Woosuk Kwon committed
215
    def get_token_ids(self) -> List[int]:
216
        return self.data.get_token_ids()
Woosuk Kwon's avatar
Woosuk Kwon committed
217

218
    def get_last_token_id(self) -> int:
219
        return self.data.get_last_token_id()
220

221
222
223
224
225
226
    def get_output_token_ids(self) -> List[int]:
        return self.data.output_token_ids

    def get_cumulative_logprob(self) -> float:
        return self.data.cumulative_logprob

227
    def get_beam_search_score(self,
228
                              length_penalty: float = 1.0,
229
230
231
232
233
234
235
236
237
238
                              seq_len: Optional[int] = None,
                              eos_token_id: Optional[int] = None) -> float:
        """Calculate the beam search score with length penalty.

        Adapted from

        https://github.com/huggingface/transformers/blob/ccb92be23def445f2afdea94c31286f84b89eb5b/src/transformers/generation/beam_search.py#L938
        """
        if seq_len is None:
            seq_len = self.get_len()
239
            # NOTE: HF implementation does not count the EOS token
240
241
242
243
244
245
            # towards the length, we align with that here for testing.
            if (eos_token_id is not None
                    and self.get_last_token_id() == eos_token_id):
                seq_len -= 1
        return self.get_cumulative_logprob() / (seq_len**length_penalty)

246
247
248
    def is_finished(self) -> bool:
        return SequenceStatus.is_finished(self.status)

249
250
251
252
    def fork(self, new_seq_id: int) -> "Sequence":
        new_seq = copy.deepcopy(self)
        new_seq.seq_id = new_seq_id
        return new_seq
253

Woosuk Kwon's avatar
Woosuk Kwon committed
254
    def __repr__(self) -> str:
255
256
257
        return (f"Sequence(seq_id={self.seq_id}, "
                f"status={self.status.name}, "
                f"num_blocks={len(self.logical_token_blocks)})")
Woosuk Kwon's avatar
Woosuk Kwon committed
258

Woosuk Kwon's avatar
Woosuk Kwon committed
259

Nick Hill's avatar
Nick Hill committed
260
261
262
263
264
265
266
267
@dataclass
class SequenceGroupState:
    """Mutable state tied to a specific sequence group"""

    # torch.Generator used in seeded sampling
    generator: Optional = None


Woosuk Kwon's avatar
Woosuk Kwon committed
268
class SequenceGroup:
269
270
271
272
273
274
275
    """A group of sequences that are generated from the same prompt.

    Args:
        request_id: The ID of the request.
        seqs: The list of sequences.
        sampling_params: The sampling parameters used to generate the outputs.
        arrival_time: The arrival time of the request.
276
        lora_request: LoRA request.
277
    """
Woosuk Kwon's avatar
Woosuk Kwon committed
278
279
280

    def __init__(
        self,
281
        request_id: str,
Woosuk Kwon's avatar
Woosuk Kwon committed
282
        seqs: List[Sequence],
283
        sampling_params: SamplingParams,
284
        arrival_time: float,
285
        lora_request: Optional[LoRARequest] = None,
Woosuk Kwon's avatar
Woosuk Kwon committed
286
    ) -> None:
287
        self.request_id = request_id
288
        self.seqs_dict = {seq.seq_id: seq for seq in seqs}
289
        self.sampling_params = sampling_params
290
291
292
293
294
        self.metrics = RequestMetrics(arrival_time=arrival_time,
                                      last_token_time=arrival_time,
                                      first_scheduled_time=None,
                                      first_token_time=None,
                                      time_in_queue=None)
295
        self.lora_request = lora_request
296
        self.prompt_logprobs: Optional[PromptLogprobs] = None
Nick Hill's avatar
Nick Hill committed
297
        self.state = SequenceGroupState()
298
299
300
301
302
303
304
305
306
307
308
309

    @property
    def prompt(self) -> str:
        # All sequences in the group should have the same prompt.
        # We use the prompt of an arbitrary sequence.
        return next(iter(self.seqs_dict.values())).prompt

    @property
    def prompt_token_ids(self) -> List[int]:
        # All sequences in the group should have the same prompt.
        # We use the prompt of an arbitrary sequence.
        return next(iter(self.seqs_dict.values())).data.prompt_token_ids
Woosuk Kwon's avatar
Woosuk Kwon committed
310

311
312
313
314
    @property
    def lora_int_id(self) -> int:
        return self.lora_request.lora_int_id if self.lora_request else 0

315
316
    def get_last_latency(self, now: float) -> float:
        """Gets last token latency for Request level timings."""
317
318
        latency = now - self.metrics.last_token_time
        self.metrics.last_token_time = now
319
320
        return latency

321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
    def maybe_set_first_token_time(self, time: float) -> None:
        """Sets the first token time for Request level timings."""
        if self.metrics.first_token_time is None:
            self.metrics.first_token_time = time

    def maybe_set_first_scheduled_time(self, time: float) -> None:
        """Sets the first scheduled time and time in queue for Request level timings."""
        if self.metrics.first_scheduled_time is None:
            self.metrics.first_scheduled_time = time
            self.metrics.time_in_queue = time - self.metrics.arrival_time

    def set_finished_time(self, time: Optional[float]) -> None:
        """Sets the finished time for Request level timings."""
        self.metrics.finished_time = time

336
337
338
339
340
341
342
343
344
345
346
347
348
349
    def get_max_num_running_seqs(self) -> int:
        """The maximum number of sequences running in parallel in the remaining
        lifetime of the request."""
        if self.sampling_params.use_beam_search:
            # For beam search, maximally there will always be `best_of` beam
            # candidates running in the future.
            return self.sampling_params.best_of
        else:
            if self.sampling_params.best_of > self.num_seqs():
                # At prompt stage, the sequence group is not yet filled up
                # and only have one sequence running. However, in the
                # generation stage, we will have `best_of` sequences running.
                return self.sampling_params.best_of
            # At sampling stages, return the number of actual sequences
350
351
            # that are not finished yet.
            return self.num_unfinished_seqs()
352

353
354
355
356
    def get_seqs(
        self,
        status: Optional[SequenceStatus] = None,
    ) -> List[Sequence]:
Woosuk Kwon's avatar
Woosuk Kwon committed
357
        if status is None:
358
            return list(self.seqs_dict.values())
Woosuk Kwon's avatar
Woosuk Kwon committed
359
        else:
360
361
362
363
            return [
                seq for seq in self.seqs_dict.values() if seq.status == status
            ]

364
365
366
367
368
    def get_unfinished_seqs(self) -> List[Sequence]:
        return [
            seq for seq in self.seqs_dict.values() if not seq.is_finished()
        ]

369
370
    def get_finished_seqs(self) -> List[Sequence]:
        return [seq for seq in self.seqs_dict.values() if seq.is_finished()]
371
372
373

    def num_seqs(self, status: Optional[SequenceStatus] = None) -> int:
        return len(self.get_seqs(status))
374

375
376
377
378
379
380
    def num_unfinished_seqs(self) -> int:
        return len(self.get_unfinished_seqs())

    def num_finished_seqs(self) -> int:
        return len(self.get_finished_seqs())

381
    def find(self, seq_id: int) -> Sequence:
382
383
384
385
386
387
388
389
390
391
392
393
394
        if seq_id not in self.seqs_dict:
            raise ValueError(f"Sequence {seq_id} not found.")
        return self.seqs_dict[seq_id]

    def add(self, seq: Sequence) -> None:
        if seq.seq_id in self.seqs_dict:
            raise ValueError(f"Sequence {seq.seq_id} already exists.")
        self.seqs_dict[seq.seq_id] = seq

    def remove(self, seq_id: int) -> None:
        if seq_id not in self.seqs_dict:
            raise ValueError(f"Sequence {seq_id} not found.")
        del self.seqs_dict[seq_id]
Woosuk Kwon's avatar
Woosuk Kwon committed
395

Woosuk Kwon's avatar
Woosuk Kwon committed
396
    def is_finished(self) -> bool:
397
        return all(seq.is_finished() for seq in self.get_seqs())
Woosuk Kwon's avatar
Woosuk Kwon committed
398

Woosuk Kwon's avatar
Woosuk Kwon committed
399
    def __repr__(self) -> str:
400
401
        return (f"SequenceGroup(request_id={self.request_id}, "
                f"sampling_params={self.sampling_params}, "
402
                f"num_seqs={len(self.seqs_dict)})")
403
404


405
class SequenceGroupMetadata:
406
407
408
409
410
411
412
413
414
    """Metadata for a sequence group. Used to create `InputMetadata`.

    Args:
        request_id: The ID of the request.
        is_prompt: Whether the request is at prompt stage.
        seq_data: The sequence data. (Seq id -> sequence data)
        sampling_params: The sampling parameters used to generate the outputs.
        block_tables: The block tables. (Seq id -> list of physical block
            numbers)
Nick Hill's avatar
Nick Hill committed
415
        state: Internal state tied to this sequence group.
416
        lora_request: LoRA request.
417
    """
418
419
420

    def __init__(
        self,
421
        request_id: str,
422
        is_prompt: bool,
423
        seq_data: Dict[int, SequenceData],
424
        sampling_params: SamplingParams,
425
        block_tables: Dict[int, List[int]],
426
        lora_request: Optional[LoRARequest] = None,
427
        computed_block_nums: Optional[List[int]] = None,
Nick Hill's avatar
Nick Hill committed
428
        state: Optional[SequenceGroupState] = None,
429
    ) -> None:
430
        self.request_id = request_id
431
        self.is_prompt = is_prompt
432
        self.seq_data = seq_data
433
434
        self.sampling_params = sampling_params
        self.block_tables = block_tables
435
        self.lora_request = lora_request
436
        self.computed_block_nums = computed_block_nums
Nick Hill's avatar
Nick Hill committed
437
        self.state = SequenceGroupState() if state is None else state
438

439
440
441
442
    @property
    def lora_int_id(self) -> int:
        return self.lora_request.lora_int_id if self.lora_request else 0

443

Zhuohan Li's avatar
Zhuohan Li committed
444
class SequenceOutput:
445
446
447
448
449
450
451
452
453
    """The model output associated with a sequence.

    Args:
        parent_seq_id: The ID of the parent sequence (for forking in beam
            search).
        output_token: The output token ID.
        logprobs: The logprobs of the output token.
            (Token id -> logP(x_i+1 | x_0, ..., x_i))
    """
454
455
456
457
458

    def __init__(
        self,
        parent_seq_id: int,
        output_token: int,
459
        logprobs: Dict[int, float],
460
461
462
463
464
465
    ) -> None:
        self.parent_seq_id = parent_seq_id
        self.output_token = output_token
        self.logprobs = logprobs

    def __repr__(self) -> str:
Zhuohan Li's avatar
Zhuohan Li committed
466
        return (f"SequenceOutput(parent_seq_id={self.parent_seq_id}, "
467
468
                f"output_token={self.output_token}, "
                f"logprobs={self.logprobs})")
Zhuohan Li's avatar
Zhuohan Li committed
469

470
    def __eq__(self, other: object) -> bool:
Zhuohan Li's avatar
Zhuohan Li committed
471
        if not isinstance(other, SequenceOutput):
Zhuohan Li's avatar
Zhuohan Li committed
472
            raise NotImplementedError()
473
        return (self.parent_seq_id == other.parent_seq_id
474
475
                and self.output_token == other.output_token
                and self.logprobs == other.logprobs)
476
477


Zhuohan Li's avatar
Zhuohan Li committed
478
479
class SequenceGroupOutput:
    """The model output associated with a sequence group."""
480
481
482

    def __init__(
        self,
Zhuohan Li's avatar
Zhuohan Li committed
483
        samples: List[SequenceOutput],
484
485
486
487
488
489
        prompt_logprobs: Optional[PromptLogprobs],
    ) -> None:
        self.samples = samples
        self.prompt_logprobs = prompt_logprobs

    def __repr__(self) -> str:
Zhuohan Li's avatar
Zhuohan Li committed
490
        return (f"SequenceGroupOutput(samples={self.samples}, "
491
492
                f"prompt_logprobs={self.prompt_logprobs})")

493
    def __eq__(self, other: object) -> bool:
Zhuohan Li's avatar
Zhuohan Li committed
494
        if not isinstance(other, SequenceGroupOutput):
495
496
497
498
            raise NotImplementedError()
        return (self.samples == other.samples
                and self.prompt_logprobs == other.prompt_logprobs)

499

Zhuohan Li's avatar
Zhuohan Li committed
500
# For each sequence group, we generate a list of SequenceOutput object,
501
# each of which contains one possible candidate for the next token.
Zhuohan Li's avatar
Zhuohan Li committed
502
SamplerOutput = List[SequenceGroupOutput]