Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
OpenDAS
ColossalAI
Commits
e239cf90
Unverified
Commit
e239cf90
authored
Mar 05, 2024
by
Luo Yihang
Committed by
GitHub
Mar 05, 2024
Browse files
[hotfix] fix typo of openmoe model source (#5403)
parent
e304e4db
Changes
4
Hide whitespace changes
Inline
Side-by-side
Showing
4 changed files
with
7 additions
and
7 deletions
+7
-7
examples/language/openmoe/benchmark/benchmark_cai.py
examples/language/openmoe/benchmark/benchmark_cai.py
+1
-1
examples/language/openmoe/benchmark/benchmark_fsdp.py
examples/language/openmoe/benchmark/benchmark_fsdp.py
+1
-1
examples/language/openmoe/infer.py
examples/language/openmoe/infer.py
+3
-3
examples/language/openmoe/train.py
examples/language/openmoe/train.py
+2
-2
No files found.
examples/language/openmoe/benchmark/benchmark_cai.py
View file @
e239cf90
...
@@ -207,7 +207,7 @@ def main():
...
@@ -207,7 +207,7 @@ def main():
coordinator
.
print_on_master
(
f
"Set plugin as
{
plugin
}
"
)
coordinator
.
print_on_master
(
f
"Set plugin as
{
plugin
}
"
)
# Build OpenMoe model
# Build OpenMoe model
repo_name
=
"hpcaitech/openmoe-"
+
args
.
model_name
repo_name
=
"hpcai
-
tech/openmoe-"
+
args
.
model_name
config
=
LlamaConfig
.
from_pretrained
(
repo_name
)
config
=
LlamaConfig
.
from_pretrained
(
repo_name
)
set_openmoe_args
(
set_openmoe_args
(
config
,
config
,
...
...
examples/language/openmoe/benchmark/benchmark_fsdp.py
View file @
e239cf90
...
@@ -53,7 +53,7 @@ def fsdp_main(rank, world_size, args):
...
@@ -53,7 +53,7 @@ def fsdp_main(rank, world_size, args):
train_loader
=
torch
.
utils
.
data
.
DataLoader
(
dataset
,
**
train_kwargs
)
train_loader
=
torch
.
utils
.
data
.
DataLoader
(
dataset
,
**
train_kwargs
)
torch
.
cuda
.
set_device
(
rank
)
torch
.
cuda
.
set_device
(
rank
)
config
=
LlamaConfig
.
from_pretrained
(
"hpcaitech/openmoe-%s"
%
args
.
model_name
)
config
=
LlamaConfig
.
from_pretrained
(
"hpcai
-
tech/openmoe-%s"
%
args
.
model_name
)
set_openmoe_args
(
set_openmoe_args
(
config
,
config
,
num_experts
=
config
.
num_experts
,
num_experts
=
config
.
num_experts
,
...
...
examples/language/openmoe/infer.py
View file @
e239cf90
...
@@ -15,19 +15,19 @@ def parse_args():
...
@@ -15,19 +15,19 @@ def parse_args():
def
inference
(
args
):
def
inference
(
args
):
tokenizer
=
T5Tokenizer
.
from_pretrained
(
"google/umt5-small"
)
tokenizer
=
T5Tokenizer
.
from_pretrained
(
"google/umt5-small"
)
if
args
.
model
==
"test"
:
if
args
.
model
==
"test"
:
config
=
LlamaConfig
.
from_pretrained
(
"hpcaitech/openmoe-base"
)
config
=
LlamaConfig
.
from_pretrained
(
"hpcai
-
tech/openmoe-base"
)
set_openmoe_args
(
config
,
set_openmoe_args
(
config
,
num_experts
=
config
.
num_experts
,
num_experts
=
config
.
num_experts
,
moe_layer_interval
=
config
.
moe_layer_interval
,
moe_layer_interval
=
config
.
moe_layer_interval
,
enable_kernel
=
True
)
enable_kernel
=
True
)
model
=
OpenMoeForCausalLM
(
config
)
model
=
OpenMoeForCausalLM
(
config
)
else
:
else
:
config
=
LlamaConfig
.
from_pretrained
(
f
"hpcaitech/openmoe-
{
args
.
model
}
"
)
config
=
LlamaConfig
.
from_pretrained
(
f
"hpcai
-
tech/openmoe-
{
args
.
model
}
"
)
set_openmoe_args
(
config
,
set_openmoe_args
(
config
,
num_experts
=
config
.
num_experts
,
num_experts
=
config
.
num_experts
,
moe_layer_interval
=
config
.
moe_layer_interval
,
moe_layer_interval
=
config
.
moe_layer_interval
,
enable_kernel
=
False
)
enable_kernel
=
False
)
model
=
OpenMoeForCausalLM
.
from_pretrained
(
f
"hpcaitech/openmoe-
{
args
.
model
}
"
,
config
=
config
)
model
=
OpenMoeForCausalLM
.
from_pretrained
(
f
"hpcai
-
tech/openmoe-
{
args
.
model
}
"
,
config
=
config
)
model
=
model
.
eval
().
bfloat16
()
model
=
model
.
eval
().
bfloat16
()
model
=
model
.
to
(
torch
.
cuda
.
current_device
())
model
=
model
.
to
(
torch
.
cuda
.
current_device
())
...
...
examples/language/openmoe/train.py
View file @
e239cf90
...
@@ -269,12 +269,12 @@ def main():
...
@@ -269,12 +269,12 @@ def main():
# Build OpenMoe model
# Build OpenMoe model
if
test_mode
:
if
test_mode
:
config
=
LlamaConfig
.
from_pretrained
(
"hpcaitech/openmoe-base"
)
config
=
LlamaConfig
.
from_pretrained
(
"hpcai
-
tech/openmoe-base"
)
config
.
hidden_size
=
128
config
.
hidden_size
=
128
config
.
intermediate_size
=
256
config
.
intermediate_size
=
256
config
.
vocab_size
=
32000
config
.
vocab_size
=
32000
else
:
else
:
repo_name
=
"hpcaitech/openmoe-"
+
args
.
model_name
repo_name
=
"hpcai
-
tech/openmoe-"
+
args
.
model_name
config
=
LlamaConfig
.
from_pretrained
(
repo_name
)
config
=
LlamaConfig
.
from_pretrained
(
repo_name
)
set_openmoe_args
(
set_openmoe_args
(
config
,
config
,
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment