Skip to content

GitLab

Unverified Commit 8c742a66 authored by Woosuk Kwon's avatar Woosuk Kwon Committed by GitHub
Browse files

[Misc] Avoid redundant copy for encoder-only models (#24012) 


Signed-off-by: default avatarWoosuk Kwon <woosuk.kwon@berkeley.edu>
parent 183a7096
Show whitespace changes
Inline Side-by-side
Showing with 7 additions and 7 deletions
vllm/v1/worker/gpu_model_runner.py
View file @ 8c742a66
...@@ -827,13 +827,13 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin): ...@@ -827,13 +827,13 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
blk_table_tensor = torch.zeros( blk_table_tensor = torch.zeros(
(num_reqs, 1), (num_reqs, 1),
dtype=torch.int32, dtype=torch.int32,
pin_memory=self.pin_memory, device=self.device,
device="cpu").to(self.device, non_blocking=True) )
slot_mapping = torch.zeros((total_num_scheduled_tokens, ), slot_mapping = torch.zeros(
dtype=torch.int32, (total_num_scheduled_tokens, ),
pin_memory=self.pin_memory, dtype=torch.int64,
device="cpu").to(self.device, device=self.device,
non_blocking=True) )
num_common_prefix_blocks = 0 num_common_prefix_blocks = 0
else: else:
blk_table = self.input_batch.block_table[kv_cache_group_id] blk_table = self.input_batch.block_table[kv_cache_group_id]
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment