[rocHPL] Update lib paths and ucx configure args

projects/rocHPL/rochpl-install.patch

@@ -131,7 +131,7 @@ index 6d6be5d..d11c01a 100644
  
  include(ROCMSetupVersion)
 diff --git a/install.sh b/install.sh
-index b30a3fb..4016a90 100755
+index b30a3fb..f23ee73 100755
 --- a/install.sh
 +++ b/install.sh
 @@ -2,7 +2,7 @@
@@ -174,7 +174,7 @@ index b30a3fb..4016a90 100755
          printf "sudo dnf install -y ${library_dependencies_fedora[*]}\n"
          ;;
  
-@@ -105,42 +106,60 @@ check_exit_code( )
+@@ -105,42 +106,61 @@ check_exit_code( )
  # Clone and build OpenMPI+UCX in rochpl/tpl
  install_openmpi( )
  {
@@ -217,9 +217,6 @@ index b30a3fb..4016a90 100755
 +  local ucx_lib64_folder=${ucx_prefix}/lib64
 +  local ompi_lib64_folder=${ompi_prefix}/lib64
 +
-+  # Create the tpl directory
-+  mkdir -p ${install_dir} && cd ${install_dir}
-+
 +  local ucx_version=1.20.0
 +  local ucx_src=${install_dir}/ucx-${ucx_version}
 +  local ucx_tarball=ucx-${ucx_version}.tar.gz
@@ -227,6 +224,9 @@ index b30a3fb..4016a90 100755
 +  local ompi_src=${install_dir}/openmpi-${ompi_version}
 +  local ompi_tarball=openmpi-${ompi_version}.tar.gz
 +
++  # Create the tpl directory
++  mkdir -p ${install_dir} && cd ${install_dir}
++
 +  # Download UCX on demand
 +  rm -rf ${ucx_src}
 +  if [ ! -f "${ucx_tarball}" ]; then
@@ -251,6 +251,7 @@ index b30a3fb..4016a90 100755
 +      [ ! -f "${ucx_lib64_folder}/libucs.so" ] || [ ! -f "${ucx_lib64_folder}/libuct.so" ]); then
 +    cd ${ucx_src};
 +    ./contrib/configure-release --prefix=${ucx_prefix} \
++      --enable-optimizations --enable-tuning \
 +      --enable-cma --enable-mt \
 +      --with-mlx5 --with-rc --with-ud --with-dc --with-dm --with-ib_hw_tm \
 +      --with-verbs=/usr/include --with-rdmacm=/usr \
@@ -265,7 +266,7 @@ index b30a3fb..4016a90 100755
    fi
  
    # Check for successful build
-@@ -152,31 +171,24 @@ install_openmpi( )
+@@ -152,31 +172,24 @@ install_openmpi( )
      exit 3
    fi
  
@@ -288,9 +289,9 @@ index b30a3fb..4016a90 100755
 -    check_exit_code 2
 -    mkdir build; cd build
 -    ../configure --prefix=${PWD}/../ --with-ucx=${PWD}/../../ucx --without-verbs --disable-man-pages --enable-mca-no-build=btl-uct
-+  export LD_LIBRARY_PATH="${ucx_lib_folder}:${ucx_lib64_folder}:${LD_LIBRARY_PATH}"
-+  export LIBRARY_PATH="${ucx_lib_folder}:${ucx_lib64_folder}:${LIBRARY_PATH}"
-+  export CPATH="${ucx_prefix}/include:${CPATH}"
++  export LD_LIBRARY_PATH="${ucx_lib_folder}:${ucx_lib64_folder}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
++  export LIBRARY_PATH="${ucx_lib_folder}:${ucx_lib64_folder}${LIBRARY_PATH:+:${LIBRARY_PATH}}"
++  export CPATH="${ucx_prefix}/include${CPATH:+:${CPATH}}"
 +
 +  # Build OpenMPI on demand
 +  if [ ! -f "${ompi_lib_folder}/libmpi.so" ] && [ ! -f "${ompi_lib64_folder}/libmpi.so" ]; then
@@ -310,21 +311,21 @@ index b30a3fb..4016a90 100755
    fi
  
    # Check for successful build
-@@ -184,6 +196,13 @@ install_openmpi( )
+@@ -184,6 +197,13 @@ install_openmpi( )
      echo "Error: OpenMPI install unsuccessful."
      exit_with_error 2
    fi
 +
-+  export LD_LIBRARY_PATH="${ompi_lib_folder}:${LD_LIBRARY_PATH}"
-+  export LIBRARY_PATH="${ompi_lib_folder}:${LIBRARY_PATH}"
-+  export CPATH="${ompi_prefix}/include:${CPATH}"
++  export LD_LIBRARY_PATH="${ompi_lib_folder}:${ompi_lib64_folder}${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
++  export LIBRARY_PATH="${ompi_lib_folder}:${ompi_lib64_folder}${LIBRARY_PATH:+:${LIBRARY_PATH}}"
++  export CPATH="${ompi_prefix}/include${CPATH:+:${CPATH}}"
 +  export OPAL_PREFIX=${ompi_prefix}
 +
 +  cd ${install_dir}/..
  }
  
  # #################################################
-@@ -232,7 +251,7 @@ enable_tracing=false
+@@ -232,7 +252,7 @@ enable_tracing=false
  # check if we have a modern version of getopt that can handle whitespace and long parameters
  getopt -T
  if [[ $? -eq 4 ]]; then
@@ -333,7 +334,7 @@ index b30a3fb..4016a90 100755
  else
    echo "Need a new version of getopt"
    exit_with_error 1
-@@ -263,6 +282,9 @@ while true; do
+@@ -263,6 +283,9 @@ while true; do
      --with-mpi)
          with_mpi=${2}
          shift 2 ;;
@@ -343,7 +344,7 @@ index b30a3fb..4016a90 100755
      --with-mpi-gtl)
          with_mpi_gtl=${2}
          shift 2 ;;
-@@ -294,9 +316,6 @@ printf "\033[32mCreating project build directory in: \033[33m${build_dir}\033[0m
+@@ -294,9 +317,6 @@ printf "\033[32mCreating project build directory in: \033[33m${build_dir}\033[0m
  # #################################################
  # prep
  # #################################################
@@ -353,7 +354,7 @@ index b30a3fb..4016a90 100755
  # Default cmake executable is called cmake
  cmake_executable=cmake
  
-@@ -347,11 +366,14 @@ pushd .
+@@ -347,11 +367,14 @@ pushd .
    if [[ "${enable_tracing}" == on || "${enable_tracing}" == true || "${enable_tracing}" == 1 || "${enable_tracing}" == enabled ]]; then
      cmake_common_options="${cmake_common_options} -DHPL_TRACING=ON"
    fi

projects/rocHPL/rochpl-scripts-bw.patch

 diff --git a/scripts/mpirun_rochpl.in b/scripts/mpirun_rochpl.in
-index 155f502..aaab1f2 100755
+index 155f502..895c708 100755
 --- a/scripts/mpirun_rochpl.in
 +++ b/scripts/mpirun_rochpl.in
 @@ -31,6 +31,9 @@ function display_help()
@@ -37,17 +37,19 @@ index 155f502..aaab1f2 100755
  # #################################################
  # global variables
  # #################################################
-@@ -101,7 +113,17 @@ filename=HPL.dat
+@@ -101,7 +113,19 @@ filename=HPL.dat
  inputfile=false
  cmdrun=false
  
 +tpl_dir=$(dirname "$(readlink -f "$0")")/../tpl
 +ompi_prefix=$tpl_dir/openmpi
 +ompi_lib_dir=$tpl_dir/openmpi/lib
++ompi_lib64_dir=$tpl_dir/openmpi/lib64
 +ucx_lib_dir=$tpl_dir/ucx/lib
++ucx_lib64_dir=$tpl_dir/ucx/lib64
 +
-+export LD_LIBRARY_PATH=$ompi_lib_dir:$ucx_lib_dir:$LD_LIBRARY_PATH
-+export OPAL_PREFIX=$ompi_prefix
++export LD_LIBRARY_PATH="${ompi_lib_dir}:${ompi_lib64_dir}:${ucx_lib_dir}:${ucx_lib64_dir}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
++export OPAL_PREFIX="${ompi_prefix}"
  devices=
 +nodes=
 +tcp_iface=p14p2
@@ -55,7 +57,7 @@ index 155f502..aaab1f2 100755
  
  # #################################################
  # MPI Args
-@@ -111,21 +133,6 @@ num_cpu_cores=$(lscpu | grep "Core(s)" | awk '{print $4}')
+@@ -111,21 +135,6 @@ num_cpu_cores=$(lscpu | grep "Core(s)" | awk '{print $4}')
  num_cpu_sockets=$(lscpu | grep Socket | awk '{print $2}')
  total_cpu_cores=$(($num_cpu_cores*$num_cpu_sockets))
  
@@ -77,7 +79,7 @@ index 155f502..aaab1f2 100755
  # #################################################
  # Parameter parsing
  # #################################################
-@@ -133,7 +140,7 @@ fi
+@@ -133,7 +142,7 @@ fi
  # check if we have a modern version of getopt that can handle whitespace and long parameters
  getopt -T
  if [[ $? -eq 4 ]]; then
@@ -86,7 +88,7 @@ index 155f502..aaab1f2 100755
  else
    echo "Need a new version of getopt"
    exit 1
-@@ -153,7 +160,7 @@ while true; do
+@@ -153,7 +162,7 @@ while true; do
          exit 0
          ;;
      --version)
@@ -95,7 +97,7 @@ index 155f502..aaab1f2 100755
          exit 0
          ;;
      -P)
-@@ -189,6 +196,15 @@ while true; do
+@@ -189,6 +198,15 @@ while true; do
      --devices)
          devices=${2}
          shift 2 ;;
@@ -111,7 +113,7 @@ index 155f502..aaab1f2 100755
      --) shift ; break ;;
      *)  echo "Unexpected command line parameter received; aborting";
          exit 1
-@@ -218,5 +234,77 @@ if [ ! -z "${devices}" ]; then
+@@ -218,5 +236,77 @@ if [ ! -z "${devices}" ]; then
    rochpl_args+=" --devices=${devices}"
  fi
  
@@ -193,7 +195,7 @@ index 155f502..aaab1f2 100755
 +fi
 \ No newline at end of file
 diff --git a/scripts/run_rochpl.in b/scripts/run_rochpl.in
-index 1522e5d..68c1958 100755
+index 1522e5d..3fa6338 100755
 --- a/scripts/run_rochpl.in
 +++ b/scripts/run_rochpl.in
 @@ -46,7 +46,7 @@ supported_distro( )
@@ -205,18 +207,19 @@ index 1522e5d..68c1958 100755
          true
          ;;
      *)  printf "This script is currently supported on Debian, Linuxmint, Ubuntu, CentOS, RHEL, Fedora and SLES\n"
-@@ -104,7 +104,9 @@ cmdrun=false
+@@ -104,7 +104,10 @@ cmdrun=false
  
  devices=
  
 -export LD_LIBRARY_PATH=${rocblas_dir}:${blas_dir}:${rocm_dir}/lib:$LD_LIBRARY_PATH
 +tpl_dir=$(dirname "$(readlink -f "$0")")/../tpl
 +ucx_lib_dir=$tpl_dir/ucx/lib
-+export LD_LIBRARY_PATH=${rocblas_dir}:${blas_dir}:$ucx_lib_dir:${rocm_dir}/lib:$LD_LIBRARY_PATH
++ucx_lib64_dir=$tpl_dir/ucx/lib64
++export LD_LIBRARY_PATH="${rocblas_dir}:${blas_dir}:${ucx_lib_dir}:${ucx_lib64_dir}:${rocm_dir}/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}"
  
  # #################################################
  # Parameter parsing
-@@ -274,7 +276,7 @@ myq=$((rank/p))
+@@ -274,7 +277,7 @@ myq=$((rank/p))
  cpulist=$(lscpu --parse=CPU,CORE,NODE | awk '!/#/' | tr ',' "\t" | sort -k 2 -g -s)
  
  #construct list of devices and their numa affinities
@@ -225,7 +228,7 @@ index 1522e5d..68c1958 100755
  
  #count the cpus per core
  threads_per_core=$(echo "${cpulist}" | grep -c ".*	0	.*")
-@@ -361,9 +363,23 @@ export OMP_NUM_THREADS=${omp_num_threads}
+@@ -361,9 +364,23 @@ export OMP_NUM_THREADS=${omp_num_threads}
  export OMP_PLACES=${omp_places}
  export OMP_PROC_BIND=true
  
@@ -250,7 +253,7 @@ index 1522e5d..68c1958 100755
  fi
  
  rochpl_args="-P ${P} -Q ${Q} -p ${p} -q ${q} -f ${frac} -it ${it}"
-@@ -374,4 +390,5 @@ else
+@@ -374,4 +391,5 @@ else
  fi
  
  #run