update qwen2-moe layout and benchmark_throughput.py

benchmarks/benchmark_throughput.py

@@ -165,9 +165,9 @@ def sample_requests(tokenizer: PreTrainedTokenizerBase,
 
 
 def run_vllm(
-    warmup_requests: List[SampleRequest],
     requests: List[SampleRequest],
     n: int,
+    num_iters_warmup: int,
     engine_args: EngineArgs,
 ) -> float:
     from vllm import LLM, SamplingParams
@@ -193,40 +193,23 @@ def run_vllm(
         lora_requests = [request.lora_request for request in requests]
 
     # warmup
-    warmup_prompts: List[TextPrompt] = []
-    warmup_sampling_params: List[SamplingParams] = []
-    for request in warmup_requests:
-        warmup_prompts.append(
-            TextPrompt(prompt=request.prompt,
-                       multi_modal_data=request.multi_modal_data))
-        warmup_sampling_params.append(
-            SamplingParams(
-                n=n,
-                temperature=1.0,
-                top_p=1.0,
-                ignore_eos=True,
-                max_tokens=request.expected_output_len,
-            ))
-        
-    print("Warming up...")
-    for _ in tqdm(range(args.num_iters_warmup), desc="Warmup iterations"):
-        llm.generate(warmup_prompts, warmup_sampling_params, use_tqdm=True)
+    warmup_sampling_params = SamplingParams(
+        n=args.n,
+        temperature=1.0,
+        top_p=1.0,
+        ignore_eos=True,
+        max_tokens=10,
+    )
+    dummy_prompt_token_ids = np.random.randint(10000, size=(1,10))
+    dummy_prompts: List[PromptType] = [{
+        "prompt_token_ids": batch
+    } for batch in dummy_prompt_token_ids.tolist()]
     
-    # dummy_prompt_token_ids = np.random.randint(10000,
-    #                                            size=(args.num_prompts,
-    #                                                  args.input_len))
-    # dummy_prompts: List[PromptType] = [{
-    #     "prompt_token_ids": batch
-    # } for batch in dummy_prompt_token_ids.tolist()]
-    
-    # def run_to_completion(profile_dir: Optional[str] = None):
-    #     llm.generate(dummy_prompts,
-    #                     sampling_params=sampling_params,
-    #                     use_tqdm=False)
-    
-    # print("Warming up...")
-    # for _ in tqdm(range(args.num_iters_warmup), desc="Warmup iterations"):
-    #     run_to_completion(profile_dir=None)
+    print("Warming up...")
+    for _ in tqdm(range(num_iters_warmup), desc="Warmup iterations"):
+        llm.generate(dummy_prompts,
+                        sampling_params=warmup_sampling_params,
+                        use_tqdm=False)
 
     use_beam_search = False
 
@@ -384,9 +367,6 @@ def main(args: argparse.Namespace):
     # Sample the requests.
     tokenizer = AutoTokenizer.from_pretrained(
         args.tokenizer, trust_remote_code=args.trust_remote_code)
-    warmup_prompt = "hi" * 10
-    warmup_requests = [(warmup_prompt, 10, 10)
-                for _ in range(1)]
     if args.dataset is None:
         vocab_size = tokenizer.vocab_size
         requests = []
@@ -442,7 +422,7 @@ def main(args: argparse.Namespace):
                     args.disable_frontend_multiprocessing,
                 ))
         else:
-            elapsed_time = run_vllm(warmup_requests, requests, args.n,
+            elapsed_time = run_vllm(requests, args.n, args.num_iters_warmup,
                                     EngineArgs.from_cli_args(args))
     elif args.backend == "hf":
         assert args.tensor_parallel_size == 1

vllm/benchmarks/benchmark_throughput.py

@@ -165,9 +165,9 @@ def sample_requests(tokenizer: PreTrainedTokenizerBase,
 
 
 def run_vllm(
-    warmup_requests: List[SampleRequest],
     requests: List[SampleRequest],
     n: int,
+    num_iters_warmup: int,
     engine_args: EngineArgs,
 ) -> float:
     from vllm import LLM, SamplingParams
@@ -193,40 +193,23 @@ def run_vllm(
         lora_requests = [request.lora_request for request in requests]
 
     # warmup
-    warmup_prompts: List[TextPrompt] = []
-    warmup_sampling_params: List[SamplingParams] = []
-    for request in warmup_requests:
-        warmup_prompts.append(
-            TextPrompt(prompt=request.prompt,
-                       multi_modal_data=request.multi_modal_data))
-        warmup_sampling_params.append(
-            SamplingParams(
-                n=n,
-                temperature=1.0,
-                top_p=1.0,
-                ignore_eos=True,
-                max_tokens=request.expected_output_len,
-            ))
-        
-    print("Warming up...")
-    for _ in tqdm(range(args.num_iters_warmup), desc="Warmup iterations"):
-        llm.generate(warmup_prompts, warmup_sampling_params, use_tqdm=True)
+    warmup_sampling_params = SamplingParams(
+        n=args.n,
+        temperature=1.0,
+        top_p=1.0,
+        ignore_eos=True,
+        max_tokens=10,
+    )
+    dummy_prompt_token_ids = np.random.randint(10000, size=(1,10))
+    dummy_prompts: List[PromptType] = [{
+        "prompt_token_ids": batch
+    } for batch in dummy_prompt_token_ids.tolist()]
     
-    # dummy_prompt_token_ids = np.random.randint(10000,
-    #                                            size=(args.num_prompts,
-    #                                                  args.input_len))
-    # dummy_prompts: List[PromptType] = [{
-    #     "prompt_token_ids": batch
-    # } for batch in dummy_prompt_token_ids.tolist()]
-    
-    # def run_to_completion(profile_dir: Optional[str] = None):
-    #     llm.generate(dummy_prompts,
-    #                     sampling_params=sampling_params,
-    #                     use_tqdm=False)
-    
-    # print("Warming up...")
-    # for _ in tqdm(range(args.num_iters_warmup), desc="Warmup iterations"):
-    #     run_to_completion(profile_dir=None)
+    print("Warming up...")
+    for _ in tqdm(range(num_iters_warmup), desc="Warmup iterations"):
+        llm.generate(dummy_prompts,
+                        sampling_params=warmup_sampling_params,
+                        use_tqdm=False)
 
     use_beam_search = False
 
@@ -384,9 +367,6 @@ def main(args: argparse.Namespace):
     # Sample the requests.
     tokenizer = AutoTokenizer.from_pretrained(
         args.tokenizer, trust_remote_code=args.trust_remote_code)
-    warmup_prompt = "hi" * 10
-    warmup_requests = [(warmup_prompt, 10, 10)
-                for _ in range(1)]
     if args.dataset is None:
         vocab_size = tokenizer.vocab_size
         requests = []
@@ -442,7 +422,7 @@ def main(args: argparse.Namespace):
                     args.disable_frontend_multiprocessing,
                 ))
         else:
-            elapsed_time = run_vllm(warmup_requests, requests, args.n,
+            elapsed_time = run_vllm(requests, args.n, args.num_iters_warmup,
                                     EngineArgs.from_cli_args(args))
     elif args.backend == "hf":
         assert args.tensor_parallel_size == 1

vllm/model_executor/models/qwen2_moe.py

@@ -553,7 +553,7 @@ class Qwen2MoeForCausalLM(nn.Module, SupportsPP):
                     weight_loader = getattr(param, "weight_loader",
                                             default_weight_loader)
                     weight_loader(param, loaded_weight)
-                loaded_params.add(name)
+            loaded_params.add(name)
 
         if self.use_llama_nn and self.quant_method is None:
             lay_key_words = [