"vscode:/vscode.git/clone" did not exist on "c61a98f52993550e42d1bded121341fb9afb2ddf"
Unverified Commit f9995398 authored by danisereb's avatar danisereb Committed by GitHub
Browse files

Add missing import of fused_topk to benchmark_moe (#32784)


Signed-off-by: default avatarDaniel Serebrenik <daserebrenik@nvidia.com>
parent e1da249c
...@@ -15,6 +15,7 @@ import ray ...@@ -15,6 +15,7 @@ import ray
import torch import torch
from ray.experimental.tqdm_ray import tqdm from ray.experimental.tqdm_ray import tqdm
from vllm.model_executor.layers.fused_moe import fused_topk
from vllm.model_executor.layers.fused_moe.config import ( from vllm.model_executor.layers.fused_moe.config import (
FusedMoEConfig, FusedMoEConfig,
FusedMoEParallelConfig, FusedMoEParallelConfig,
...@@ -200,6 +201,8 @@ def benchmark_config( ...@@ -200,6 +201,8 @@ def benchmark_config(
block_shape=block_quant_shape, block_shape=block_quant_shape,
) )
deep_gemm_experts = None
if use_deep_gemm:
deep_gemm_experts = mk.FusedMoEModularKernel( deep_gemm_experts = mk.FusedMoEModularKernel(
prepare_finalize=MoEPrepareAndFinalizeNoEP(), prepare_finalize=MoEPrepareAndFinalizeNoEP(),
fused_experts=TritonOrDeepGemmExperts( fused_experts=TritonOrDeepGemmExperts(
...@@ -210,9 +213,10 @@ def benchmark_config( ...@@ -210,9 +213,10 @@ def benchmark_config(
intermediate_size_per_partition=shard_intermediate_size, intermediate_size_per_partition=shard_intermediate_size,
num_local_experts=num_experts, num_local_experts=num_experts,
activation="silu", activation="silu",
parallel_config=FusedMoEParallelConfig.make_no_parallel(), moe_parallel_config=FusedMoEParallelConfig.make_no_parallel(),
in_dtype=init_dtype, in_dtype=init_dtype,
routing_method=RoutingMethodType.TopK, routing_method=RoutingMethodType.TopK,
device="cuda",
), ),
quant_config=quant_config, quant_config=quant_config,
), ),
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment