Skip to content

GitLab

Unverified Commit 1a41c9e9 authored by Umang Yadav's avatar Umang Yadav Committed by GitHub
Browse files

Add clean up for CI cache and change keys for caching (#1680) 

CI changes to improve github cache management
parent c5eee1a3
Hide whitespace changes
Inline Side-by-side
Showing with 168 additions and 89 deletions
.github/workflows/ci.yaml
View file @ 1a41c9e9
name: migraphx name: migraphx
on: [push, pull_request] on:
pull_request:
push:
branches:
- develop
- master
- 'release/**'
jobs: jobs:
cancel: cancel:
...@@ -17,40 +24,24 @@ jobs: ...@@ -17,40 +24,24 @@ jobs:
- name: Free space - name: Free space
run: | run: |
sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws* /usr/local/lib/heroku sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws* /usr/local/lib/heroku
du . --max-depth=1 -h
ls -la
cd /usr/local
du . --max-depth=1 -h
ls -la
cd /usr/local/lib
echo $(pwd)
du . --max-depth=1 -h
ls -la
- uses: actions/checkout@v3 - uses: actions/checkout@v3
# In this step, this action saves a list of existing images, # In this step, this action saves a list of existing images,
# the cache is created without them in the post run. # the cache is created without them in the post run.
# It also restores the cache if it exists. # It also restores the cache if it exists.
# name: Docker Layer Caching2
- uses: jpribyl/action-docker-layer-caching@v0.1.1 - uses: jpribyl/action-docker-layer-caching@v0.1.1
# Ignore the failure of a step and avoid terminating the job. # Ignore the failure of a step and avoid terminating the job.
continue-on-error: true continue-on-error: true
- name: Prepare timestamp - name: Restore cache files for tidy
id: cache_timestamp uses: actions/cache/restore@v3
shell: bash id: tidy_restore
run: echo timestamp="$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_OUTPUT
- name: Cache files for tidy
uses: pat-s/always-upload-cache@v3.0.11
with: with:
path: tidy-cache path: tidy-cache
key: tidy-cache-${{ steps.cache_timestamp.outputs.timestamp }} key: tidy-cache-${{ github.ref }}
restore-keys: | restore-keys: tidy-cache-
tidy-cache-${{ steps.cache_timestamp.outputs.timestamp }}
tidy-cache-
- name: Build the Docker image - name: Build the Docker image
run: docker build . --file hip-clang.docker --tag migraphx run: docker build . --file hip-clang.docker --tag migraphx
...@@ -70,6 +61,25 @@ jobs: ...@@ -70,6 +61,25 @@ jobs:
.. ..
make -j2 -k onnx-proto tf-proto tidy make -j2 -k onnx-proto tf-proto tidy
# GH actions can not update existing cache, as a workaround clear cache and then save it
- name: Clear tidy cache before saving
if: ${{ steps.tidy_restore.outputs.cache-hit }}
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh extension install actions/gh-actions-cache --pin v1.0.1
gh actions-cache delete ${{ steps.tidy_restore.outputs.cache-matched-key }} --confirm
continue-on-error: true
- name: Save cache files for tidy
uses: actions/cache/save@v3
if: always()
with:
path: tidy-cache
key: tidy-cache-${{ github.ref }}
cppcheck: cppcheck:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
...@@ -85,19 +95,13 @@ jobs: ...@@ -85,19 +95,13 @@ jobs:
# Ignore the failure of a step and avoid terminating the job. # Ignore the failure of a step and avoid terminating the job.
continue-on-error: true continue-on-error: true
- name: Prepare timestamp - name: Restore cache files for cppcheck
id: cache_timestamp id: cppcheck_restore
shell: bash uses: actions/cache/restore@v3
run: echo timestamp="$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_OUTPUT
- name: Cache files for cppcheck
uses: pat-s/always-upload-cache@v2.1.3
with: with:
path: cppcheck-cache path: cppcheck-cache
key: cppcheck-cache-${{ hashFiles('cppcheck.rules', 'CMakeLists.txt') }}-${{ steps.cache_timestamp.outputs.timestamp }} key: cppcheck-cache-${{ hashFiles('cppcheck.rules', 'CMakeLists.txt') }}-${{ github.ref }}
restore-keys: | restore-keys: cppcheck-cache-${{ hashFiles('cppcheck.rules', 'CMakeLists.txt') }}-
cppcheck-cache-${{ hashFiles('cppcheck.rules', 'CMakeLists.txt') }}-${{ steps.cache_timestamp.outputs.timestamp }}
cppcheck-cache-${{ hashFiles('cppcheck.rules', 'CMakeLists.txt') }}-
- name: Build the Docker image - name: Build the Docker image
run: docker build . --file hip-clang.docker --tag migraphx run: docker build . --file hip-clang.docker --tag migraphx
...@@ -114,6 +118,25 @@ jobs: ...@@ -114,6 +118,25 @@ jobs:
.. ..
make -j2 cppcheck make -j2 cppcheck
# GH actions can not update existing cache, as a workaround clear cache and then save it
- name: Clear cppcheck cache before saving
if: ${{ steps.cppcheck_restore.outputs.cache-hit }}
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh extension install actions/gh-actions-cache --pin v1.0.1
gh actions-cache delete ${{ steps.cppcheck_restore.outputs.cache-matched-key }} --confirm
continue-on-error: true
- name: Save cache files for cppcheck
uses: actions/cache/save@v3
if: always()
with:
path: cppcheck-cache
key: cppcheck-cache-${{ hashFiles('cppcheck.rules', 'CMakeLists.txt') }}-${{ github.ref }}
format: format:
runs-on: ubuntu-20.04 runs-on: ubuntu-20.04
...@@ -206,8 +229,13 @@ jobs: ...@@ -206,8 +229,13 @@ jobs:
- codecov - codecov
steps: steps:
- name: Free space - name: Free space and install rbuild, lld
run: sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws* /usr/local/lib/heroku run: |
sudo rm -rf /usr/local/android /usr/share/dotnet /usr/local/share/boost /opt/ghc /usr/local/share/chrom* /usr/share/swift /usr/local/julia* /usr/local/lib/android /usr/local/graalvm /usr/local/aws* /usr/local/lib/heroku
sudo apt-get install -y lld
python -m pip install --upgrade pip
pip install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
- uses: actions/checkout@v3 - uses: actions/checkout@v3
- name: Set up Python - name: Set up Python
uses: actions/setup-python@v4 uses: actions/setup-python@v4
...@@ -217,36 +245,25 @@ jobs: ...@@ -217,36 +245,25 @@ jobs:
# Ignore the failure of a step and avoid terminating the job. # Ignore the failure of a step and avoid terminating the job.
continue-on-error: true continue-on-error: true
uses: actions/cache@v3 uses: actions/cache@v3
id: deps_cache
with: with:
# This path is specific to Ubuntu # This path is specific to Ubuntu
path: ${{ github.workspace }}/cget path: ${{ github.workspace }}/cget
# Look to see if there is a cache hit for the corresponding requirements file # Look to see if there is a cache hit for the corresponding requirements file
key: key: ${{ matrix.os }}-cget-4-${{ hashFiles('requirements.txt', 'dev-requirements.txt') }}
${{ matrix.os }}-cget-4-${{ hashFiles('requirements.txt', 'dev-requirements.txt') }} restore-keys: ${{ matrix.os }}-cget-4-
${{ matrix.os }}-cget-4-
- name: Install dependencies - name: Install dependencies
run: | if: steps.deps_cache.outputs.cache-hit != 'true'
python -m pip install --upgrade pip run: rbuild prepare -d cget -s gh
pip install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
rbuild prepare -d cget -s gh
sudo apt-get install -y lld
- name: Prepare timestamp
id: cache_timestamp
shell: bash
run: echo timestamp="$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_OUTPUT
- name: Cache files for ccache - name: Restore cache files for ccache
# Ignore the failure of a step and avoid terminating the job. uses: actions/cache/restore@v3
continue-on-error: true id: ccache_restore
uses: pat-s/always-upload-cache@v2.1.3
with: with:
path: ccache path: ${{ github.workspace }}/ccache
key: ${{ matrix.os }}-${{ matrix.configuration }}-ccache-${{ steps.cache_timestamp.outputs.timestamp }} key: ${{ matrix.os }}-${{ matrix.configuration }}-ccache-${{ github.ref }}
restore-keys: | restore-keys: ${{ matrix.os }}-${{ matrix.configuration }}-ccache-
${{ matrix.os }}-${{ matrix.configuration }}-ccache-${{ steps.cache_timestamp.outputs.timestamp }}
${{ matrix.os }}-${{ matrix.configuration }}-ccache-
- name: Build and test - name: Build and test
env: env:
...@@ -266,6 +283,23 @@ jobs: ...@@ -266,6 +283,23 @@ jobs:
-DCMAKE_SHARED_LINKER_FLAGS='-fuse-ld=lld' -DCMAKE_SHARED_LINKER_FLAGS='-fuse-ld=lld'
${{ github.workspace }}/cget/bin/ccache -s ${{ github.workspace }}/cget/bin/ccache -s
# GH actions can not update existing cache, as a workaround clear cache and then save it
- name: Clear ccache cache before saving
if: ${{ steps.ccache_restore.outputs.cache-hit }}
shell: bash
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
gh extension install actions/gh-actions-cache --pin v1.0.1
gh actions-cache delete ${{ steps.ccache_restore.outputs.cache-matched-key }} --confirm
- name: Save cache files for ccache
uses: actions/cache/save@v3
if: always()
with:
path: ${{ github.workspace }}/ccache
key: ${{ matrix.os }}-${{ matrix.configuration }}-ccache-${{ github.ref }}
- name: Upload code coverage - name: Upload code coverage
if: "matrix.configuration == 'codecov'" if: "matrix.configuration == 'codecov'"
env: env:
...@@ -309,6 +343,7 @@ jobs: ...@@ -309,6 +343,7 @@ jobs:
uses: actions/setup-python@v4 uses: actions/setup-python@v4
with: with:
python-version: 3.7 python-version: 3.7
- name: Cache dependencies - name: Cache dependencies
# Ignore the failure of a step and avoid terminating the job. # Ignore the failure of a step and avoid terminating the job.
continue-on-error: true continue-on-error: true
...@@ -317,9 +352,8 @@ jobs: ...@@ -317,9 +352,8 @@ jobs:
# This path is specific to Ubuntu # This path is specific to Ubuntu
path: ${{ github.workspace }}/cget path: ${{ github.workspace }}/cget
# Look to see if there is a cache hit for the corresponding requirements file # Look to see if there is a cache hit for the corresponding requirements file
key: key: ${{ matrix.os }}-cget-4-${{ hashFiles('requirements.txt', 'dev-requirements.txt') }}
${{ matrix.os }}-cget-4-${{ hashFiles('requirements.txt', 'dev-requirements.txt') }} restore-keys: ${{ matrix.os }}-cget-4-
${{ matrix.os }}-cget-4-
- name: Install dependencies - name: Install dependencies
...@@ -328,22 +362,15 @@ jobs: ...@@ -328,22 +362,15 @@ jobs:
pip install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz pip install https://github.com/RadeonOpenCompute/rbuild/archive/master.tar.gz
rbuild prepare -d cget -s gh rbuild prepare -d cget -s gh
sudo apt-get install -y lld sudo apt-get install -y lld
- name: Prepare timestamp
id: cache_timestamp
shell: bash
run: echo timestamp="$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_OUTPUT
- name: Cache files for ccache - name: Restore cache files for ccache
# Ignore the failure of a step and avoid terminating the job. id: ccache_restore_fpga
continue-on-error: true uses: actions/cache/restore@v3
uses: pat-s/always-upload-cache@v2.1.3
with: with:
path: ccache path: ${{ github.workspace }}/ccache
key: ${{ matrix.os }}-${{ matrix.configuration }}-ccache-${{ steps.cache_timestamp.outputs.timestamp }} key: ${{ matrix.os }}-${{ matrix.configuration }}-ccache-${{ github.ref }}
restore-keys: | restore-keys: ${{ matrix.os }}-${{ matrix.configuration }}-ccache-
${{ matrix.os }}-${{ matrix.configuration }}-ccache-${{ steps.cache_timestamp.outputs.timestamp }}
${{ matrix.os }}-${{ matrix.configuration }}-ccache-
- name: Build and test - name: Build and test
env: env:
CMAKE_PREFIX_PATH: ${{ github.workspace }}/cget CMAKE_PREFIX_PATH: ${{ github.workspace }}/cget
...@@ -363,17 +390,36 @@ jobs: ...@@ -363,17 +390,36 @@ jobs:
-DMIGRAPHX_ENABLE_FPGA=On -DMIGRAPHX_ENABLE_FPGA=On
${{ github.workspace }}/cget/bin/ccache -s ${{ github.workspace }}/cget/bin/ccache -s
#- name: Upload code coverage # this is a workaround, with GH actions can not update existing cache
# if: "matrix.configuration == 'codecov'" - name: Clear ccache cache before saving
# env: if: ${{ steps.ccache_restore_fpga.outputs.cache-hit }}
# CODECOV_TOKEN: "8545af1c-f90b-4345-92a5-0d075503ca56" shell: bash
# run: | env:
# sudo apt-get install -y lcov GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# cd build run: |
# lcov --directory . --capture --output-file $(pwd)/coverage.info gh extension install actions/gh-actions-cache
# lcov --remove $(pwd)/coverage.info '/usr/*' --output-file $(pwd)/coverage.info gh actions-cache delete ${{ steps.ccache_restore_fpga.outputs.cache-matched-key }} --confirm
# lcov --list $(pwd)/coverage.info continue-on-error: true
# curl -Os https://uploader.codecov.io/latest/linux/codecov
# chmod +x codecov - name: Save cache files for ccache
# ./codecov -t ${CODECOV_TOKEN} uses: actions/cache/save@v3
# echo "Uploaded" if: always()
with:
path: ${{ github.workspace }}/ccache
key: ${{ matrix.os }}-${{ matrix.configuration }}-ccache-${{ github.ref }}
#- name: Upload code coverage
# if: "matrix.configuration == 'codecov'"
# env:
# CODECOV_TOKEN: "8545af1c-f90b-4345-92a5-0d075503ca56"
# run: |
# sudo apt-get install -y lcov
# cd build
# lcov --directory . --capture --output-file $(pwd)/coverage.info
# lcov --remove $(pwd)/coverage.info '/usr/*' --output-file $(pwd)/coverage.info
# lcov --list $(pwd)/coverage.info
# curl -Os https://uploader.codecov.io/latest/linux/codecov
# chmod +x codecov
# ./codecov -t ${CODECOV_TOKEN}
# echo "Uploaded"
.github/workflows/clean-closed-pr-caches.yaml 0 → 100644
View file @ 1a41c9e9
name: Cleanup caches of closed PR
on:
pull_request:
types:
- closed
jobs:
cleanup:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v3
- name: Cleanup
run: |
gh extension install actions/gh-actions-cache --pin v1.0.1
REPO=${{ github.repository }}
BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge"
echo "Fetching list of cache key"
cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH | cut -f 1 )
## Setting this to not fail the workflow while deleting cache keys.
set +e
echo "Deleting caches..."
for cacheKey in $cacheKeysForPR
do
gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm
done
echo "Done"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment