Unverified Commit e0f5cc68 authored by Jeff Rasley's avatar Jeff Rasley Committed by GitHub
Browse files

add skip reqs flag (#133)

parent 259f894a
...@@ -22,6 +22,7 @@ hostfile (hostfile: /job/hostfile). If no hostfile exists, will only install loc ...@@ -22,6 +22,7 @@ hostfile (hostfile: /job/hostfile). If no hostfile exists, will only install loc
-m, --pip_mirror Use the specified pip mirror (default: the default pip mirror) -m, --pip_mirror Use the specified pip mirror (default: the default pip mirror)
-H, --hostfile Path to MPI-style hostfile (default: /job/hostfile) -H, --hostfile Path to MPI-style hostfile (default: /job/hostfile)
-a, --apex_commit Install a specific commit hash of apex, instead of the one deepspeed points to -a, --apex_commit Install a specific commit hash of apex, instead of the one deepspeed points to
-k, --skip_requirements Skip installing DeepSpeed requirements
-h, --help This help text -h, --help This help text
""" """
} }
...@@ -36,6 +37,7 @@ entire_dlts_job=1 ...@@ -36,6 +37,7 @@ entire_dlts_job=1
hostfile=/job/hostfile hostfile=/job/hostfile
pip_mirror="" pip_mirror=""
apex_commit="" apex_commit=""
skip_requirements=0
while [[ $# -gt 0 ]] while [[ $# -gt 0 ]]
do do
...@@ -71,6 +73,10 @@ case $key in ...@@ -71,6 +73,10 @@ case $key in
shift shift
shift shift
;; ;;
-k|--skip_requirements)
skip_requirements=1;
shift
;;
-H|--hostfile) -H|--hostfile)
hostfile=$2 hostfile=$2
if [ ! -f $2 ]; then if [ ! -f $2 ]; then
...@@ -122,8 +128,10 @@ if [ ! -f $hostfile ]; then ...@@ -122,8 +128,10 @@ if [ ! -f $hostfile ]; then
local_only=1 local_only=1
fi fi
# Ensure dependencies are installed locally if [ "$skip_requirements" == "0" ]; then
$PIP_SUDO $PIP_INSTALL -r requirements.txt # Ensure dependencies are installed locally
$PIP_SUDO $PIP_INSTALL -r requirements.txt
fi
# Build wheels # Build wheels
if [ "$third_party_install" == "1" ]; then if [ "$third_party_install" == "1" ]; then
...@@ -172,7 +180,9 @@ else ...@@ -172,7 +180,9 @@ else
pdsh -w $hosts "if [ -d $tmp_wheel_path ]; then rm $tmp_wheel_path/*.whl; else mkdir -pv $tmp_wheel_path; fi" pdsh -w $hosts "if [ -d $tmp_wheel_path ]; then rm $tmp_wheel_path/*.whl; else mkdir -pv $tmp_wheel_path; fi"
pdcp -w $hosts requirements.txt ${tmp_wheel_path}/ pdcp -w $hosts requirements.txt ${tmp_wheel_path}/
pdsh -w $hosts "$PIP_SUDO $PIP_INSTALL -r ${tmp_wheel_path}/requirements.txt" if [ "$skip_requirements" == "0" ]; then
pdsh -w $hosts "$PIP_SUDO $PIP_INSTALL -r ${tmp_wheel_path}/requirements.txt"
fi
if [ "$third_party_install" == "1" ]; then if [ "$third_party_install" == "1" ]; then
pdsh -w $hosts "$PIP_SUDO pip uninstall -y apex" pdsh -w $hosts "$PIP_SUDO pip uninstall -y apex"
pdcp -w $hosts third_party/apex/dist/apex*.whl $tmp_wheel_path/ pdcp -w $hosts third_party/apex/dist/apex*.whl $tmp_wheel_path/
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment