Skip to content

GitLab

Unverified Commit 482e8d61 authored by Chris Austen's avatar Chris Austen Committed by GitHub
Browse files

Cache docker when using Github actions (#2196) 

Adjust Github Action flows to reduce the workload per commit by caching the docker image once so all jobs can use it
parent 0bb8508b
Hide whitespace changes
Inline Side-by-side
Showing with 119 additions and 64 deletions
.github/workflows/ci.yaml
View file @ 482e8d61
......@@ -8,6 +8,9 @@ on:
- master
- 'release/**'
env:
DOCKER_USER: ${{secrets.DOCKERHUB_USERID}}
DOCKER_TOKEN: ${{secrets.DOCKERHUB_TOKEN}}
jobs:
cancel:
......@@ -17,23 +20,93 @@ jobs:
uses: styfle/cancel-workflow-action@0.11.0
with:
access_token: ${{ github.token }}
check_image:
name: Check if image exists in registry
runs-on: ubuntu-latest
outputs:
imageexists: ${{ steps.check_image.outputs.imageexists }}
imagetag: ${{ steps.image_hash.outputs.imagetag }}
imageexists_sles: ${{ steps.check_image.outputs.imageexists_sles }}
imagetag_sles: ${{ steps.image_hash.outputs.imagetag_sles }}
steps:
- name: Checkout Code
uses: actions/checkout@v3
- name: Create Image Tag
id: image_hash
run: |
echo "imagetag=rocm/migraphx-private:hip-clang-${{hashFiles('**/hip-clang.docker', '**/*requirements.txt', '**/install_prereqs.sh', '**/rbuild.ini')}}" >> $GITHUB_OUTPUT
echo "imagetag_sles=rocm/migraphx-sles-private:hip-clang-${{hashFiles('**/tools/docker/sles.docker', '**/*requirements.txt', '**/install_prereqs.sh', '**/rbuild.ini')}}" >> $GITHUB_OUTPUT
- name: Check if image is built already
id: check_image
env:
DOCKERIMAGE: ${{ steps.image_hash.outputs.imagetag }}
DOCKERIMAGE_SLES: ${{ steps.image_hash.outputs.imagetag_sles }}
run: |
echo $DOCKER_TOKEN | docker login -u $DOCKER_USER --password-stdin
if [[ "$(docker manifest inspect $DOCKERIMAGE 2> /dev/null)" != "" ]]; then
echo "imageexists=true" >> $GITHUB_OUTPUT
echo "Image already exists, skip building available"
else
echo "imageexists=false" >> $GITHUB_OUTPUT
echo "Tag does not exist, build and publishing required"
fi
if [[ "$(docker manifest inspect $DOCKERIMAGE_SLES 2> /dev/null)" != "" ]]; then
echo "imageexists_sles=true" >> $GITHUB_OUTPUT
echo "SLES Image already exists, skip building available"
else
echo "imageexists_sles=false" >> $GITHUB_OUTPUT
echo "SLES Tag does not exist, build and publishing required"
fi
build_image:
name: Build image
runs-on: ROCM-Ubuntu
needs: check_image
if: ${{ needs.check_image.outputs.imageexists != 'true' }}
steps:
- uses: actions/checkout@v3
- name: Build and publish
env:
DOCKERIMAGE: ${{ needs.check_image.outputs.imagetag }}
run: |
echo $DOCKER_TOKEN | docker login -u $DOCKER_USER --password-stdin
docker build . --file hip-clang.docker --tag $DOCKERIMAGE;
docker push $DOCKERIMAGE;
build_SLES_image:
name: Build SLES image
runs-on: ROCM-Ubuntu
needs: check_image
if: ${{ needs.check_image.outputs.imageexists_sles != 'true' }}
steps:
- uses: actions/checkout@v3
- name: Build and publish SLES
env:
DOCKERIMAGE_SLES: ${{ needs.check_image.outputs.imagetag_sles }}
run: |
echo $DOCKER_TOKEN | docker login -u $DOCKER_USER --password-stdin
docker build . --file tools/docker/sles.docker --tag $DOCKERIMAGE_SLES;
docker push $DOCKERIMAGE_SLES;
tidy:
runs-on: ROCM-Ubuntu
needs: [ build_image, check_image ]
env:
DOCKERIMAGE: ${{ needs.check_image.outputs.imagetag }}
if: ${{ !cancelled() && (needs.build_image.result == 'success' || needs.build_image.result == 'skipped') }}
steps:
- uses: actions/checkout@v3
# In this step, this action saves a list of existing images,
# the cache is created without them in the post run.
# It also restores the cache if it exists.
- name: Docker layer cache
uses: jpribyl/action-docker-layer-caching@v0.1.1
with:
key: docker-layer-caching-migraphx-${{hashFiles('hip-clang.docker', '**/*requirements.txt', '**/install_prereqs.sh', 'rbuild.ini')}}
restore-keys:
docker-layer-caching-migraphx-
# Ignore the failure of a step and avoid terminating the job.
continue-on-error: true
- name: Restore cache files for tidy
uses: actions/cache/restore@v3
id: tidy_restore
......@@ -41,13 +114,13 @@ jobs:
path: tidy-cache
key: tidy-cache-${{ github.ref }}
restore-keys: tidy-cache-
- name: Build the Docker image
- name: Docker Login
run: |
docker build . --file hip-clang.docker --tag migraphx
echo $DOCKER_TOKEN | docker login -u $DOCKER_USER --password-stdin
- name: Clang tidy
shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data migraphx bash < {0}"
- name: Clang Tidy
shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data $DOCKERIMAGE bash < {0}"
run: |
mkdir build
cd build
......@@ -84,21 +157,14 @@ jobs:
cppcheck:
runs-on: ROCM-Ubuntu
needs: [ build_image, check_image ]
env:
DOCKERIMAGE: ${{ needs.check_image.outputs.imagetag }}
if: ${{ !cancelled() && (needs.build_image.result == 'success' || needs.build_image.result == 'skipped') }}
steps:
- uses: actions/checkout@v3
# In this step, this action saves a list of existing images,
# the cache is created without them in the post run.
# It also restores the cache if it exists.
- name: Docker layer cache
uses: jpribyl/action-docker-layer-caching@v0.1.1
with:
key: docker-layer-caching-migraphx-${{hashFiles('hip-clang.docker', '**/*requirements.txt', '**/install_prereqs.sh', 'rbuild.ini')}}
restore-keys:
docker-layer-caching-migraphx-
# Ignore the failure of a step and avoid terminating the job.
continue-on-error: true
- name: Restore cache files for cppcheck
id: cppcheck_restore
uses: actions/cache/restore@v3
......@@ -107,11 +173,12 @@ jobs:
key: cppcheck-cache-${{ hashFiles('cppcheck.rules', 'CMakeLists.txt') }}-${{ github.ref }}
restore-keys: cppcheck-cache-${{ hashFiles('cppcheck.rules', 'CMakeLists.txt') }}-
- name: Build the Docker image
run: docker build . --file hip-clang.docker --tag migraphx
- name: Docker Login
run: |
echo $DOCKER_TOKEN | docker login -u $DOCKER_USER --password-stdin
- name: Cppcheck
shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data migraphx bash < {0}"
shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data $DOCKERIMAGE bash < {0}"
run: |
mkdir build
cd build
......@@ -142,29 +209,23 @@ jobs:
format:
runs-on: ROCM-Ubuntu
runs-on: ubuntu-latest
needs: [ build_image, check_image ]
env:
DOCKERIMAGE: ${{ needs.check_image.outputs.imagetag }}
if: ${{ !cancelled() && (needs.build_image.result == 'success' || needs.build_image.result == 'skipped') }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
# In this step, this action saves a list of existing images,
# the cache is created without them in the post run.
# It also restores the cache if it exists.
- name: Docker layer cache
uses: jpribyl/action-docker-layer-caching@v0.1.1
with:
key: docker-layer-caching-migraphx-${{hashFiles('hip-clang.docker', '**/*requirements.txt', '**/install_prereqs.sh', 'rbuild.ini')}}
restore-keys:
docker-layer-caching-migraphx-
# Ignore the failure of a step and avoid terminating the job.
continue-on-error: true
- name: Build the Docker image
run: docker build . --file hip-clang.docker --tag migraphx
- name: Docker Login
run: |
echo $DOCKER_TOKEN | docker login -u $DOCKER_USER --password-stdin
- name: Check formatting
shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data migraphx bash < {0}"
shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data $DOCKERIMAGE bash < {0}"
run: |
set -e
git config --global --add safe.directory /data
......@@ -172,26 +233,16 @@ jobs:
sles:
runs-on: ROCM-Ubuntu
needs: [ build_SLES_image, check_image ]
env:
DOCKERIMAGE_SLES: ${{ needs.check_image.outputs.imagetag_sles }}
if: ${{ !cancelled() && (needs.build_SLES_image.result == 'success' || needs.build_SLES_image.result == 'skipped') }}
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 0
# In this step, this action saves a list of existing images,
# the cache is created without them in the post run.
# It also restores the cache if it exists.
- name: Docker layer cache
uses: jpribyl/action-docker-layer-caching@v0.1.1
with:
key: docker-layer-caching-migraphx-sles-${{hashFiles('hip-clang.docker', '**/*requirements.txt', '**/install_prereqs.sh', 'rbuild.ini')}}
restore-keys:
docker-layer-caching-migraphx-sles-
# Ignore the failure of a step and avoid terminating the job.
continue-on-error: true
- name: Build the Docker image
run: docker build . --file tools/docker/sles.docker --tag migraphx-sles
- name: Restore cache files for ccache
uses: actions/cache/restore@v3
id: ccache_restore
......@@ -200,8 +251,12 @@ jobs:
key: ccache-sles-${{ github.ref }}
restore-keys: ccache-sles-
- name: Docker Login
run: |
echo $DOCKER_TOKEN | docker login -u $DOCKER_USER --password-stdin
- name: Build migraphx
shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data migraphx-sles bash < {0}"
shell: bash -c "docker run -i -v=$GITHUB_WORKSPACE:/data -w /data $DOCKERIMAGE_SLES bash < {0}"
run: |
set -e
export CCACHE_COMPRESSLEVEL=10
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment