update benchmark_throughput.py

benchmarks/benchmark_throughput.py

@@ -287,17 +287,15 @@ def main(args: argparse.Namespace):
         raise ValueError(f"Unknown backend: {args.backend}")
     total_num_tokens = sum(prompt_len + output_len
                            for _, prompt_len, output_len in requests)
-    print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
-          f"{total_num_tokens / elapsed_time:.2f} tokens/s")
     
-    # if args.dataset is None:
-    #     total_out_tokens = args.output_len * args.num_prompts
-    # else:
-    #     total_out_tokens = sum(output_len for _, _, output_len in requests) 
-    # print(f"Latency: {elapsed_time:.2f} s")
-    # print(f"All Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
-    #       f"{total_num_tokens / elapsed_time:.2f} tokens/s")
-    # print(f"Generate Throughput: {total_out_tokens / elapsed_time:.2f} tokens/s")
+    if args.dataset is None:
+        total_out_tokens = args.output_len * args.num_prompts
+    else:
+        total_out_tokens = sum(output_len for _, _, output_len in requests) 
+    print(f"Latency: {elapsed_time:.2f} s")
+    print(f"All Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
+          f"{total_num_tokens / elapsed_time:.2f} tokens/s")
+    print(f"Generate Throughput: {total_out_tokens / elapsed_time:.2f} tokens/s")
 
 
     # Output JSON results if specified
@@ -492,4 +490,4 @@ if __name__ == "__main__":
         if args.tokenizer != args.model:
             raise ValueError("Tokenizer must be the same as the model for MII "
                              "backend.")
-    main(args)
+    main(args)
\ No newline at end of file

vllm/benchmark_throughput.py

@@ -287,17 +287,15 @@ def main(args: argparse.Namespace):
         raise ValueError(f"Unknown backend: {args.backend}")
     total_num_tokens = sum(prompt_len + output_len
                            for _, prompt_len, output_len in requests)
-    print(f"Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
-          f"{total_num_tokens / elapsed_time:.2f} tokens/s")
     
-    # if args.dataset is None:
-    #     total_out_tokens = args.output_len * args.num_prompts
-    # else:
-    #     total_out_tokens = sum(output_len for _, _, output_len in requests) 
-    # print(f"Latency: {elapsed_time:.2f} s")
-    # print(f"All Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
-    #       f"{total_num_tokens / elapsed_time:.2f} tokens/s")
-    # print(f"Generate Throughput: {total_out_tokens / elapsed_time:.2f} tokens/s")
+    if args.dataset is None:
+        total_out_tokens = args.output_len * args.num_prompts
+    else:
+        total_out_tokens = sum(output_len for _, _, output_len in requests) 
+    print(f"Latency: {elapsed_time:.2f} s")
+    print(f"All Throughput: {len(requests) / elapsed_time:.2f} requests/s, "
+          f"{total_num_tokens / elapsed_time:.2f} tokens/s")
+    print(f"Generate Throughput: {total_out_tokens / elapsed_time:.2f} tokens/s")
 
 
     # Output JSON results if specified

vllm/model_executor/layers/quantization/awq.py

@@ -195,7 +195,7 @@ class AWQLinearMethod(LinearMethodBase):
         else:
             padding_group=0
         
-        if m<4096: 
+        if m<20000: 
             out = ops.awq_gemm(reshaped_x,
                             qweight,
                             zeros_and_scales,