"vllm/vscode:/vscode.git/clone" did not exist on "3a5de7d2d6e65b6580c3ceb905334843a7b6dd6f"
Unverified Commit abfcdcdf authored by Woosuk Kwon's avatar Woosuk Kwon Committed by GitHub
Browse files

[V1][Minor] Avoid frequently creating ConstantList (#12653)



A small optimization to avoid creating a new `ConstantList` every time `request.kv_block_hashes` is used.
Signed-off-by: default avatarWoosuk Kwon <woosuk.kwon@berkeley.edu>
parent e497f334
......@@ -64,6 +64,7 @@ class Request:
# Cache the computed kv block hashes of the request to avoid
# recomputing.
self._kv_block_hashes: List[BlockHashType] = []
self.kv_block_hashes = ConstantList(self._kv_block_hashes)
# Read-only views
# Prevent directly appending to the these lists since
......@@ -121,13 +122,9 @@ class Request:
num_tokens = self.mm_positions[input_id]["length"]
return num_tokens
@property
def kv_block_hashes(self) -> ConstantList["BlockHashType"]:
# Prevent directly appending to the kv_block_hashes.
return ConstantList(self._kv_block_hashes)
def set_kv_block_hashes(self, value: List["BlockHashType"]) -> None:
self._kv_block_hashes = value
self.kv_block_hashes = ConstantList(self._kv_block_hashes)
def append_kv_block_hashes(self, block_hash: "BlockHashType") -> None:
self._kv_block_hashes.append(block_hash)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment