Update rbuild to handle dependencies better and remove unused dependencies (#825)

* Add rbuild.ini file

* Skip rbuild step

* Use rbuild for gh runners

* Update ini file

* Invalidate cache

* Use full path for ccache

* Use master
Co-authored-by: mvermeulen <5479696+mvermeulen@users.noreply.github.com>

.github/workflows/ci.yaml

@@ -166,19 +166,16 @@ jobs:
         # This path is specific to Ubuntu
         path: ${{ github.workspace }}/cget
         # Look to see if there is a cache hit for the corresponding requirements file
-        key: ${{ matrix.os }}-cget-2-${{ hashFiles('requirements.txt') }}
+        key: 
+          ${{ matrix.os }}-cget-4-${{ hashFiles('requirements.txt', 'dev-requirements.txt') }}
+          ${{ matrix.os }}-cget-4-
 
 
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip
-        pip install cget
-        cget install pfultz2/rocm-recipes
-        cget ignore ROCmSoftwarePlatform/rocBLAS ROCmSoftwarePlatform/MIOpen
-        cget install -f requirements.txt
-        cget install oneapi-src/oneDNN@v1.7
-        cget install facebook/zstd@v1.4.5 -X subdir -DCMAKE_DIR=build/cmake
-        cget install ccache@v4.1
+        pip install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
+        rbuild prepare -d cget -s gh
     - name: Prepare timestamp
       id: cache_timestamp
       shell: cmake -P {0}
@@ -205,19 +202,13 @@ jobs:
       run: |
         echo "leak:dnnl::impl::malloc" > suppressions.txt
         export LSAN_OPTIONS="suppressions=$(pwd)/suppressions.txt"
-        mkdir build
-        cd build
-        cmake .. \
-          -DCMAKE_C_COMPILER_LAUNCHER=${{ github.workspace }}/cget/bin/ccache \
-          -DCMAKE_CXX_COMPILER_LAUNCHER=${{ github.workspace }}/cget/bin/ccache \
+        rbuild build -d cget -s gh -t check \
           -DCMAKE_BUILD_TYPE=${{matrix.configuration}} \
-          -DMIGRAPHX_ENABLE_CPU=On \
           -DMIGRAPHX_ENABLE_PYTHON=${{matrix.configuration == 'release' && 'On' || 'Off'}} \
           -DCMAKE_CXX_FLAGS_DEBUG="-g1 -Os -fdebug-prefix-map=$PWD=. -fdebug-types-section -fno-omit-frame-pointer ${{matrix.os != 'ubuntu-16.04' && '-fsanitize-address-use-after-scope' || ''}} -fsanitize=undefined,address -fno-sanitize-recover=undefined,address" \
           -DCMAKE_CXX_FLAGS_CODECOV="-g1 -Og -fdebug-prefix-map=$PWD=. -fdebug-types-section -fprofile-arcs -ftest-coverage -fno-omit-frame-pointer" \
           -DCMAKE_EXE_LINKER_FLAGS='-fuse-ld=gold' \
           -DCMAKE_SHARED_LINKER_FLAGS='-fuse-ld=gold'
-        make -j2 check
         ${{ github.workspace }}/cget/bin/ccache -s
 
     - name: Upload code coverage

Dockerfile

@@ -53,6 +53,7 @@ ENV LANG=C.UTF-8
 # Install dependencies
 ADD dev-requirements.txt /dev-requirements.txt
 ADD requirements.txt /requirements.txt
+ADD rbuild.ini /rbuild.ini
 
 COPY ./tools/install_prereqs.sh /
 RUN /install_prereqs.sh /usr/local / && rm /install_prereqs.sh

dev-requirements.txt

 pfultz2/rocm-recipes
+facebook/zstd@v1.4.5 -X subdir -DCMAKE_DIR=build/cmake
+ccache@v4.1
 danmar/cppcheck@4a8a78a9258fd56bc21e55b5b49a0f09bc8fa750 -DHAVE_RULES=1
-ROCm-Developer-Tools/HIP@roc-3.0.0
+RadeonOpenCompute/rocm-cmake@ececd2eccae4d01e7ec154efe90ac43ebf4df317 --build
 -f requirements.txt

hip-clang.docker

@@ -44,9 +44,6 @@ RUN ldconfig
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 
-# Install rbuild
-RUN pip3 install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
-
 # Install yapf
 RUN pip3 install yapf==0.28.0
 
@@ -57,6 +54,7 @@ RUN pip3 install -r /doc-requirements.txt
 # Install dependencies
 ADD dev-requirements.txt /dev-requirements.txt
 ADD requirements.txt /requirements.txt
+ADD rbuild.ini /rbuild.ini
 
 COPY ./tools/install_prereqs.sh /
 RUN /install_prereqs.sh /usr/local / && rm /install_prereqs.sh

rbuild.ini

+[main]
+cxx = ${rocm_path}/llvm/bin/clang++
+cc = ${rocm_path}/llvm/bin/clang
+deps =
+    pfultz2/rocm-recipes
+    -f requirements.txt
+
+[gh]
+ignore = danmar/cppcheck
+deps =
+    -f dev-requirements.txt
+    oneapi-src/oneDNN@v1.7
+define =
+    CMAKE_C_COMPILER_LAUNCHER=${deps_dir}/bin/ccache
+    CMAKE_CXX_COMPILER_LAUNCHER=${deps_dir}/bin/ccache
+    MIGRAPHX_ENABLE_CPU=On
+
+[develop]
+cxx = ${rocm_path}/llvm/bin/clang++
+cc = ${rocm_path}/llvm/bin/clang
+deps =
+    -f dev-requirements.txt
+    oneapi-src/oneDNN@v1.7
+define =
+    CMAKE_C_COMPILER_LAUNCHER=${deps_dir}/bin/ccache
+    CMAKE_CXX_COMPILER_LAUNCHER=${deps_dir}/bin/ccache
+    MIGRAPHX_ENABLE_CPU=On
\ No newline at end of file

requirements.txt

 google/protobuf@v3.11.0 -DCMAKE_POSITION_INDEPENDENT_CODE=On -X subdir -Dprotobuf_BUILD_TESTS=Off
-RadeonOpenCompute/rocm-cmake@ececd2eccae4d01e7ec154efe90ac43ebf4df317 --build
-ROCmSoftwarePlatform/rocBLAS@abd98a2b48b29326ebaef471630786a548622c06
-ROCmSoftwarePlatform/MIOpen@2.4.0
 nlohmann/json@v3.8.0
 blaze,https://bitbucket.org/blaze-lib/blaze/get/f0755dea0e03.tar.gz -X header -DHEADER_DIR=blaze
 half,https://github.com/pfultz2/half/archive/1.12.0.tar.gz -X header -H sha256:0a08660b68abb176ebc2a0cdf8de46e3182a7f46c66443bb80dbfaaec98cf969

tools/install_prereqs.sh

@@ -12,23 +12,15 @@ PREFIX=/usr/local
 REQ_FILE_DIR=""
 if [ "$#" -ge 2 ]; then
   PREFIX=$1
-  REQ_FILE_DIR=$2
+  cd $2
 elif [ "$#" -eq 1 ]; then
   PREFIX=$1
 fi
 
 echo "Dependencies are install at $PREFIX"
 
-# Manually ignore rocm dependencies
-cget -p $PREFIX ignore \
-    RadeonOpenCompute/clang-ocl \
-    ROCm-Developer-Tools/HIP \
-    ROCmSoftwarePlatform/MIOpen \
-    ROCmSoftwarePlatform/MIOpenGEMM \
-    ROCmSoftwarePlatform/rocBLAS
-cget -p $PREFIX init --cxx /opt/rocm/llvm/bin/clang++ --cc /opt/rocm/llvm/bin/clang
-cget -p $PREFIX install -f ${REQ_FILE_DIR}dev-requirements.txt
-cget -p $PREFIX install oneapi-src/oneDNN@v1.7
+# Install deps with rbuild
+rbuild prepare -d $PREFIX -s develop
 
 # install onnx package for unit tests
 pip3 install onnx==1.8.1 numpy==1.18.5 typing==3.7.4 pytest==6.0.1 packaging==16.8