Skip to content

GitLab

"mmdet3d/vscode:/vscode.git/clone" did not exist on "809d1a9b009cb4333f9414427a079854f7f50852"
Commit bbd6d051 authored by baberabb's avatar baberabb
Browse files

add auto batching

parent 37f10cad
Hide whitespace changes
Inline Side-by-side
Showing with 0 additions and 5 deletions
lm_eval/models/vllm_causallms.py
View file @ bbd6d051
...@@ -36,11 +36,6 @@ class VLLM(LM): ...@@ -36,11 +36,6 @@ class VLLM(LM):
): ):
super().__init__() super().__init__()
assert "cuda" in device or device is None, "vLLM only supports CUDA" assert "cuda" in device or device is None, "vLLM only supports CUDA"
if batch_size == "auto":
eval_logger.info(
"vllm does not support auto selection. Setting batch_size to 8"
)
batch_size = 8
self.model = LLM( self.model = LLM(
model=pretrained, model=pretrained,
gpu_memory_utilization=gpu_memory_utilization, gpu_memory_utilization=gpu_memory_utilization,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment