Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
Menu
Open sidebar
evt_fugx1
dcu_megatron
Commits
535c37b6
Commit
535c37b6
authored
May 16, 2025
by
silencealiang
Browse files
update model parameters format
parent
83fab71e
Changes
11
Hide whitespace changes
Inline
Side-by-side
Showing
11 changed files
with
241 additions
and
91 deletions
+241
-91
dcu_megatron/training/initialize.py
dcu_megatron/training/initialize.py
+1
-1
examples/deepseek_v3/train_deepseekv3_671B_128nodes.sh
examples/deepseek_v3/train_deepseekv3_671B_128nodes.sh
+24
-9
examples/deepseek_v3/train_deepseekv3_671B_1nodes.sh
examples/deepseek_v3/train_deepseekv3_671B_1nodes.sh
+24
-9
examples/deepseek_v3/train_deepseekv3_671B_4nodes.sh
examples/deepseek_v3/train_deepseekv3_671B_4nodes.sh
+24
-9
examples/gpt3/train_gpt_567B_128nodes.sh
examples/gpt3/train_gpt_567B_128nodes.sh
+24
-9
examples/gpt3/train_gpt_567B_1nodes.sh
examples/gpt3/train_gpt_567B_1nodes.sh
+24
-9
examples/llama/train_llama2_7b_1nodes.sh
examples/llama/train_llama2_7b_1nodes.sh
+24
-9
examples/mixtral/train_mixtral_8x22B_1nodes.sh
examples/mixtral/train_mixtral_8x22B_1nodes.sh
+24
-9
examples/mixtral/train_mixtral_8x22B_8nodes.sh
examples/mixtral/train_mixtral_8x22B_8nodes.sh
+24
-9
examples/mixtral/train_mixtral_8x7B_1nodes.sh
examples/mixtral/train_mixtral_8x7B_1nodes.sh
+24
-9
examples/mixtral/train_mixtral_8x7B_4nodes.sh
examples/mixtral/train_mixtral_8x7B_4nodes.sh
+24
-9
No files found.
dcu_megatron/training/initialize.py
View file @
535c37b6
...
@@ -96,7 +96,7 @@ def _initialize_distributed(get_embedding_ranks, get_position_embedding_ranks):
...
@@ -96,7 +96,7 @@ def _initialize_distributed(get_embedding_ranks, get_position_embedding_ranks):
print
(
"> initializing torch distributed ..."
,
flush
=
True
)
print
(
"> initializing torch distributed ..."
,
flush
=
True
)
# Manually set the device ids.
# Manually set the device ids.
if
device_count
>
0
:
if
device_count
>
0
:
torch
.
cuda
.
set_device
(
args
.
local_rank
)
torch
.
cuda
.
set_device
(
args
.
local_rank
%
device_count
)
device_id
=
torch
.
device
(
f
'cuda:
{
args
.
local_rank
}
'
)
device_id
=
torch
.
device
(
f
'cuda:
{
args
.
local_rank
}
'
)
else
:
else
:
device_id
=
None
device_id
=
None
...
...
examples/deepseek_v3/train_deepseekv3_671B_128nodes.sh
View file @
535c37b6
...
@@ -429,14 +429,29 @@ elif [[ $profiling == "hip" ]]; then
...
@@ -429,14 +429,29 @@ elif [[ $profiling == "hip" ]]; then
fi
fi
#for hygon cpu
#for hygon cpu
export
CUDA_VISIBLE_DEVICES
=
0,1,2,3,4,5,6,7
case
${
LOCAL_RANK
}
in
case
${
LOCAL_RANK
}
in
0
)
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
0
)
1
)
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
0
2
)
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
3
)
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
1
)
4
)
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
1
5
)
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
6
)
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
2
)
7
)
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
2
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
3
)
export
HIP_VISIBLE_DEVICES
=
3
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
4
)
export
HIP_VISIBLE_DEVICES
=
4
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
5
)
export
HIP_VISIBLE_DEVICES
=
5
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
6
)
export
HIP_VISIBLE_DEVICES
=
6
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
7
)
export
HIP_VISIBLE_DEVICES
=
7
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
esac
esac
\ No newline at end of file
examples/deepseek_v3/train_deepseekv3_671B_1nodes.sh
View file @
535c37b6
...
@@ -429,14 +429,29 @@ elif [[ $profiling == "hip" ]]; then
...
@@ -429,14 +429,29 @@ elif [[ $profiling == "hip" ]]; then
fi
fi
#for hygon cpu
#for hygon cpu
export
CUDA_VISIBLE_DEVICES
=
0,1,2,3,4,5,6,7
case
${
LOCAL_RANK
}
in
case
${
LOCAL_RANK
}
in
0
)
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
0
)
1
)
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
0
2
)
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
3
)
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
1
)
4
)
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
1
5
)
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
6
)
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
2
)
7
)
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
2
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
3
)
export
HIP_VISIBLE_DEVICES
=
3
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
4
)
export
HIP_VISIBLE_DEVICES
=
4
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
5
)
export
HIP_VISIBLE_DEVICES
=
5
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
6
)
export
HIP_VISIBLE_DEVICES
=
6
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
7
)
export
HIP_VISIBLE_DEVICES
=
7
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
esac
esac
\ No newline at end of file
examples/deepseek_v3/train_deepseekv3_671B_4nodes.sh
View file @
535c37b6
...
@@ -429,14 +429,29 @@ elif [[ $profiling == "hip" ]]; then
...
@@ -429,14 +429,29 @@ elif [[ $profiling == "hip" ]]; then
fi
fi
#for hygon cpu
#for hygon cpu
export
CUDA_VISIBLE_DEVICES
=
0,1,2,3,4,5,6,7
case
${
LOCAL_RANK
}
in
case
${
LOCAL_RANK
}
in
0
)
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
0
)
1
)
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
0
2
)
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
3
)
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
1
)
4
)
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
1
5
)
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
6
)
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
2
)
7
)
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
2
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
3
)
export
HIP_VISIBLE_DEVICES
=
3
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
4
)
export
HIP_VISIBLE_DEVICES
=
4
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
5
)
export
HIP_VISIBLE_DEVICES
=
5
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
6
)
export
HIP_VISIBLE_DEVICES
=
6
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
7
)
export
HIP_VISIBLE_DEVICES
=
7
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
esac
esac
\ No newline at end of file
examples/gpt3/train_gpt_567B_128nodes.sh
View file @
535c37b6
...
@@ -164,14 +164,29 @@ elif [[ $profiling == "hip" ]]; then
...
@@ -164,14 +164,29 @@ elif [[ $profiling == "hip" ]]; then
fi
fi
#for hygon cpu
#for hygon cpu
export
CUDA_VISIBLE_DEVICES
=
0,1,2,3,4,5,6,7
case
${
LOCAL_RANK
}
in
case
${
LOCAL_RANK
}
in
0
)
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
0
)
1
)
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
0
2
)
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
3
)
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
1
)
4
)
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
1
5
)
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
6
)
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
2
)
7
)
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
2
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
3
)
export
HIP_VISIBLE_DEVICES
=
3
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
4
)
export
HIP_VISIBLE_DEVICES
=
4
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
5
)
export
HIP_VISIBLE_DEVICES
=
5
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
6
)
export
HIP_VISIBLE_DEVICES
=
6
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
7
)
export
HIP_VISIBLE_DEVICES
=
7
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
esac
esac
\ No newline at end of file
examples/gpt3/train_gpt_567B_1nodes.sh
View file @
535c37b6
...
@@ -164,14 +164,29 @@ elif [[ $profiling == "hip" ]]; then
...
@@ -164,14 +164,29 @@ elif [[ $profiling == "hip" ]]; then
fi
fi
#for hygon cpu
#for hygon cpu
export
CUDA_VISIBLE_DEVICES
=
0,1,2,3,4,5,6,7
case
${
LOCAL_RANK
}
in
case
${
LOCAL_RANK
}
in
0
)
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
0
)
1
)
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
0
2
)
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
3
)
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
1
)
4
)
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
1
5
)
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
6
)
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
2
)
7
)
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
2
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
3
)
export
HIP_VISIBLE_DEVICES
=
3
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
4
)
export
HIP_VISIBLE_DEVICES
=
4
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
5
)
export
HIP_VISIBLE_DEVICES
=
5
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
6
)
export
HIP_VISIBLE_DEVICES
=
6
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
7
)
export
HIP_VISIBLE_DEVICES
=
7
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
esac
esac
\ No newline at end of file
examples/llama/train_llama2_7b_1nodes.sh
View file @
535c37b6
...
@@ -158,14 +158,29 @@ elif [[ $profiling == "hip" ]]; then
...
@@ -158,14 +158,29 @@ elif [[ $profiling == "hip" ]]; then
fi
fi
#for hygon cpu
#for hygon cpu
export
CUDA_VISIBLE_DEVICES
=
0,1,2,3,4,5,6,7
case
${
LOCAL_RANK
}
in
case
${
LOCAL_RANK
}
in
0
)
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
0
)
1
)
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
0
2
)
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
3
)
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
1
)
4
)
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
1
5
)
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
6
)
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
2
)
7
)
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
2
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
3
)
export
HIP_VISIBLE_DEVICES
=
3
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
4
)
export
HIP_VISIBLE_DEVICES
=
4
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
5
)
export
HIP_VISIBLE_DEVICES
=
5
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
6
)
export
HIP_VISIBLE_DEVICES
=
6
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
7
)
export
HIP_VISIBLE_DEVICES
=
7
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
esac
esac
\ No newline at end of file
examples/mixtral/train_mixtral_8x22B_1nodes.sh
View file @
535c37b6
...
@@ -167,14 +167,29 @@ elif [[ $profiling == "hip" ]]; then
...
@@ -167,14 +167,29 @@ elif [[ $profiling == "hip" ]]; then
fi
fi
#for hygon cpu
#for hygon cpu
export
CUDA_VISIBLE_DEVICES
=
0,1,2,3,4,5,6,7
case
${
LOCAL_RANK
}
in
case
${
LOCAL_RANK
}
in
0
)
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
0
)
1
)
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
0
2
)
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
3
)
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
1
)
4
)
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
1
5
)
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
6
)
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
2
)
7
)
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
2
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
3
)
export
HIP_VISIBLE_DEVICES
=
3
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
4
)
export
HIP_VISIBLE_DEVICES
=
4
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
5
)
export
HIP_VISIBLE_DEVICES
=
5
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
6
)
export
HIP_VISIBLE_DEVICES
=
6
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
7
)
export
HIP_VISIBLE_DEVICES
=
7
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
esac
esac
\ No newline at end of file
examples/mixtral/train_mixtral_8x22B_8nodes.sh
View file @
535c37b6
...
@@ -167,14 +167,29 @@ elif [[ $profiling == "hip" ]]; then
...
@@ -167,14 +167,29 @@ elif [[ $profiling == "hip" ]]; then
fi
fi
#for hygon cpu
#for hygon cpu
export
CUDA_VISIBLE_DEVICES
=
0,1,2,3,4,5,6,7
case
${
LOCAL_RANK
}
in
case
${
LOCAL_RANK
}
in
0
)
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
0
)
1
)
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
0
2
)
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
3
)
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
1
)
4
)
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
1
5
)
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
6
)
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
2
)
7
)
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
2
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
3
)
export
HIP_VISIBLE_DEVICES
=
3
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
4
)
export
HIP_VISIBLE_DEVICES
=
4
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
5
)
export
HIP_VISIBLE_DEVICES
=
5
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
6
)
export
HIP_VISIBLE_DEVICES
=
6
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
7
)
export
HIP_VISIBLE_DEVICES
=
7
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
esac
esac
\ No newline at end of file
examples/mixtral/train_mixtral_8x7B_1nodes.sh
View file @
535c37b6
...
@@ -167,14 +167,29 @@ elif [[ $profiling == "hip" ]]; then
...
@@ -167,14 +167,29 @@ elif [[ $profiling == "hip" ]]; then
fi
fi
#for hygon cpu
#for hygon cpu
export
CUDA_VISIBLE_DEVICES
=
0,1,2,3,4,5,6,7
case
${
LOCAL_RANK
}
in
case
${
LOCAL_RANK
}
in
0
)
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
0
)
1
)
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
0
2
)
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
3
)
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
1
)
4
)
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
1
5
)
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
6
)
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
2
)
7
)
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
2
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
3
)
export
HIP_VISIBLE_DEVICES
=
3
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
4
)
export
HIP_VISIBLE_DEVICES
=
4
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
5
)
export
HIP_VISIBLE_DEVICES
=
5
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
6
)
export
HIP_VISIBLE_DEVICES
=
6
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
7
)
export
HIP_VISIBLE_DEVICES
=
7
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
esac
esac
\ No newline at end of file
examples/mixtral/train_mixtral_8x7B_4nodes.sh
View file @
535c37b6
...
@@ -167,14 +167,29 @@ elif [[ $profiling == "hip" ]]; then
...
@@ -167,14 +167,29 @@ elif [[ $profiling == "hip" ]]; then
fi
fi
#for hygon cpu
#for hygon cpu
export
CUDA_VISIBLE_DEVICES
=
0,1,2,3,4,5,6,7
case
${
LOCAL_RANK
}
in
case
${
LOCAL_RANK
}
in
0
)
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
0
)
1
)
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
0
2
)
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
numactl
--cpunodebind
=
0
--membind
=
0
${
APP
}
;;
3
)
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
1
)
4
)
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
1
5
)
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
numactl
--cpunodebind
=
1
--membind
=
1
${
APP
}
;;
6
)
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
2
)
7
)
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
export
HIP_VISIBLE_DEVICES
=
2
numactl
--cpunodebind
=
2
--membind
=
2
${
APP
}
;;
3
)
export
HIP_VISIBLE_DEVICES
=
3
numactl
--cpunodebind
=
3
--membind
=
3
${
APP
}
;;
4
)
export
HIP_VISIBLE_DEVICES
=
4
numactl
--cpunodebind
=
4
--membind
=
4
${
APP
}
;;
5
)
export
HIP_VISIBLE_DEVICES
=
5
numactl
--cpunodebind
=
5
--membind
=
5
${
APP
}
;;
6
)
export
HIP_VISIBLE_DEVICES
=
6
numactl
--cpunodebind
=
6
--membind
=
6
${
APP
}
;;
7
)
export
HIP_VISIBLE_DEVICES
=
7
numactl
--cpunodebind
=
7
--membind
=
7
${
APP
}
;;
esac
esac
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment
