[Benchmark] Add cpu options to bench scripts (#3915)

c013d32c · Zedong Peng · GitHub · 11dd6ebb · c013d32c · c013d32c
Unverified Commit c013d32c authored Apr 10, 2024 by Zedong Peng Committed by GitHub Apr 09, 2024
Hide whitespace changes
Inline Side-by-side

Showing with 4 additions and 4 deletions

benchmarks/benchmark_latency.py benchmarks/benchmark_latency.py +2 -2

benchmarks/benchmark_throughput.py benchmarks/benchmark_throughput.py +2 -2

No files found.
--- a/benchmarks/benchmark_latency.py
+++ b/benchmarks/benchmark_latency.py
@@ -169,8 +169,8 @@ if __name__ == '__main__':
        "--device",
        type=str,
        default="cuda",
-        choices=["cuda"],
-        help='device type for vLLM execution, supporting CUDA only currently.')
+        choices=["cuda", "cpu"],
+        help='device type for vLLM execution, supporting CUDA and CPU.')
    parser.add_argument('--block-size',
                        type=int,
                        default=16,

--- a/benchmarks/benchmark_throughput.py
+++ b/benchmarks/benchmark_throughput.py
@@ -329,8 +329,8 @@ if __name__ == "__main__":
        "--device",
        type=str,
        default="cuda",
-        choices=["cuda"],
-        help='device type for vLLM execution, supporting CUDA only currently.')
+        choices=["cuda", "cpu"],
+        help='device type for vLLM execution, supporting CUDA and CPU.')
    parser.add_argument(
        "--enable-prefix-caching",
        action='store_true',