Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
ox696c
ktransformers
Commits
ee24a270
Commit
ee24a270
authored
Feb 04, 2025
by
Azure
Browse files
update v3 single gpu rule yaml;
parent
907251c7
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
12 additions
and
6 deletions
+12
-6
ktransformers/local_chat.py
ktransformers/local_chat.py
+1
-2
ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat.yaml
ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat.yaml
+11
-4
No files found.
ktransformers/local_chat.py
View file @
ee24a270
...
...
@@ -32,8 +32,7 @@ custom_models = {
ktransformer_rules_dir
=
os
.
path
.
dirname
(
os
.
path
.
abspath
(
__file__
))
+
"/optimize/optimize_rules/"
default_optimize_rules
=
{
"DeepseekV2ForCausalLM"
:
ktransformer_rules_dir
+
"DeepSeek-V2-Chat.yaml"
,
# "DeepseekV3ForCausalLM": ktransformer_rules_dir + "DeepSeek-V2-Chat.yaml",
"DeepseekV3ForCausalLM"
:
ktransformer_rules_dir
+
"DeepSeek-V3-Chat-multi-gpu.yaml"
,
"DeepseekV3ForCausalLM"
:
ktransformer_rules_dir
+
"DeepSeek-V3-Chat.yaml"
,
"Qwen2MoeForCausalLM"
:
ktransformer_rules_dir
+
"Qwen2-57B-A14B-Instruct.yaml"
,
"LlamaForCausalLM"
:
ktransformer_rules_dir
+
"Internlm2_5-7b-Chat-1m.yaml"
,
"MixtralForCausalLM"
:
ktransformer_rules_dir
+
"Mixtral.yaml"
,
...
...
ktransformers/optimize/optimize_rules/DeepSeek-V3-Chat.yaml
View file @
ee24a270
-
match
:
class
:
ktransformers.models.modeling_deepseek.DeepseekV3
Yarn
RotaryEmbedding
class
:
ktransformers.models.modeling_deepseek
_v3
.DeepseekV3RotaryEmbedding
replace
:
class
:
ktransformers.operators.RoPE.
Yarn
RotaryEmbedding
class
:
ktransformers.operators.RoPE.RotaryEmbedding
V3
kwargs
:
generate_device
:
"
cuda"
prefill_device
:
"
cuda"
...
...
@@ -17,12 +17,19 @@
prefill_op
:
"
KLinearTorch"
-
match
:
name
:
"
^model
\\
.layers
\\
..*
\\
.mlp$"
class
:
ktransformers.models.modeling_deepseek.DeepseekV
2
MoE
class
:
ktransformers.models.modeling_deepseek
_v3
.DeepseekV
3
MoE
replace
:
class
:
ktransformers.operators.experts.KDeepseekV
2
MoE
# mlp module with custom forward function
class
:
ktransformers.operators.experts.KDeepseekV
3
MoE
# mlp module with custom forward function
kwargs
:
generate_device
:
"
cuda"
prefill_device
:
"
cuda"
-
match
:
class
:
ktransformers.models.modeling_deepseek_v3.MoEGate
replace
:
class
:
ktransformers.operators.gate.KMoEGate
kwargs
:
generate_device
:
"
cuda:0"
prefill_device
:
"
cuda:0"
-
match
:
name
:
"
^model
\\
.layers
\\
..*
\\
.mlp
\\
.experts$"
replace
:
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment