"vscode:/vscode.git/clone" did not exist on "cdeec2e6067613c501f82463d54e420097f49750"
Unverified Commit 829b9a62 authored by Lucas Wilkinson's avatar Lucas Wilkinson Committed by GitHub
Browse files

[Perf] Dont create unnecessary pooling params (#22876)


Signed-off-by: default avatarLucas Wilkinson <lwilkins@redhat.com>
parent 540d54ca
......@@ -341,13 +341,13 @@ class GPUModelRunner(LoRAModelRunnerMixin, KVConnectorModelRunnerMixin):
model_kwargs = dict[str, Any]()
num_reqs = self.input_batch.num_reqs
pooling_params = self.input_batch.pooling_metadata.pooling_params
num_pooling_reqs = len(pooling_params)
num_pooling_reqs = len(self.input_batch.pooling_params)
if num_pooling_reqs == 0:
return model_kwargs
pooling_params = self.input_batch.pooling_metadata.pooling_params
assert num_pooling_reqs == num_reqs
token_type_id_requests = dict[int, Any]()
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment