Skip to content

GitLab

Unverified Commit d9e6ce45 authored by moto's avatar moto Committed by GitHub
Browse files

Refactor Cache bust mechanism and bust on daily basis (#742) 

This PR refactors cache generation mechanism by introducing dedicated command
and bust cache on daily basis.

At this moment, Windows unittest job for 3.6 and 3.7 are broken because of
broken scipy but the environment is cached this persists until the next week.

As we have nightly build, we do not need to keep cache for one week.
parent b8ddeb35
Hide whitespace changes
Inline Side-by-side
Showing with 48 additions and 72 deletions
.circleci/config.yml
View file @ d9e6ce45
...@@ -20,6 +20,12 @@ executors: ...@@ -20,6 +20,12 @@ executors:
shell: bash.exe shell: bash.exe
commands: commands:
generate_cache_key:
description: "Generates a cache key file that changes daily"
steps:
- run:
name: Generate cache key
command: echo "$(date +"%Y-%m-%d")" > .cachekey
designate_upload_channel: designate_upload_channel:
description: "inserts the correct upload channel into ${BASH_ENV}" description: "inserts the correct upload channel into ${BASH_ENV}"
steps: steps:
...@@ -80,20 +86,17 @@ jobs: ...@@ -80,20 +86,17 @@ jobs:
resource_class: small resource_class: small
steps: steps:
- checkout - checkout
- run: - generate_cache_key
name: Generate cache key
# This will refresh cache on daily
command: echo "$(date +"%Y-%U-%d")" > .circleci-daily
- restore_cache: - restore_cache:
keys: keys:
- tp-nix-{{ checksum ".circleci-daily" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }} - tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
- run: - run:
command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only
- save_cache: - save_cache:
key: tp-nix-{{ checksum ".circleci-daily" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }} key: tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
paths: paths:
- third_party/tmp - third_party/tmp
...@@ -376,21 +379,18 @@ jobs: ...@@ -376,21 +379,18 @@ jobs:
- checkout - checkout
- attach_workspace: - attach_workspace:
at: third_party at: third_party
- run: - generate_cache_key
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache: - restore_cache:
keys: keys:
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
- run: - run:
name: Setup name: Setup
command: .circleci/unittest/linux/scripts/setup_env.sh command: .circleci/unittest/linux/scripts/setup_env.sh
- save_cache: - save_cache:
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
paths: paths:
- conda - conda
...@@ -419,21 +419,18 @@ jobs: ...@@ -419,21 +419,18 @@ jobs:
- checkout - checkout
- attach_workspace: - attach_workspace:
at: third_party at: third_party
- run: - generate_cache_key
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache: - restore_cache:
keys: keys:
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
- run: - run:
name: Setup name: Setup
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
- save_cache: - save_cache:
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
paths: paths:
- conda - conda
...@@ -457,21 +454,18 @@ jobs: ...@@ -457,21 +454,18 @@ jobs:
name: windows-cpu name: windows-cpu
steps: steps:
- checkout - checkout
- run: - generate_cache_key
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache: - restore_cache:
keys: keys:
- env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - env-v3-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
- run: - run:
name: Setup name: Setup
command: .circleci/unittest/windows/scripts/setup_env.sh command: .circleci/unittest/windows/scripts/setup_env.sh
- save_cache: - save_cache:
key: env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} key: env-v3-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
paths: paths:
- conda - conda
...@@ -496,21 +490,18 @@ jobs: ...@@ -496,21 +490,18 @@ jobs:
CUDA_VERSION: "10.1" CUDA_VERSION: "10.1"
steps: steps:
- checkout - checkout
- run: - generate_cache_key
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache: - restore_cache:
keys: keys:
- env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
- run: - run:
name: Setup name: Setup
command: .circleci/unittest/windows/scripts/setup_env.sh command: .circleci/unittest/windows/scripts/setup_env.sh
- save_cache: - save_cache:
key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
paths: paths:
- conda - conda
...@@ -534,21 +525,18 @@ jobs: ...@@ -534,21 +525,18 @@ jobs:
resource_class: medium resource_class: medium
steps: steps:
- checkout - checkout
- run: - generate_cache_key
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache: - restore_cache:
keys: keys:
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
- run: - run:
name: Setup name: Setup
command: .circleci/unittest/linux/scripts/setup_env.sh command: .circleci/unittest/linux/scripts/setup_env.sh
- save_cache: - save_cache:
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
paths: paths:
- conda - conda
......
.circleci/config.yml.in
View file @ d9e6ce45
...@@ -20,6 +20,12 @@ executors: ...@@ -20,6 +20,12 @@ executors:
shell: bash.exe shell: bash.exe
commands: commands:
generate_cache_key:
description: "Generates a cache key file that changes daily"
steps:
- run:
name: Generate cache key
command: echo "$(date +"%Y-%m-%d")" > .cachekey
designate_upload_channel: designate_upload_channel:
description: "inserts the correct upload channel into ${BASH_ENV}" description: "inserts the correct upload channel into ${BASH_ENV}"
steps: steps:
...@@ -80,20 +86,17 @@ jobs: ...@@ -80,20 +86,17 @@ jobs:
resource_class: small resource_class: small
steps: steps:
- checkout - checkout
- run: - generate_cache_key
name: Generate cache key
# This will refresh cache on daily
command: echo "$(date +"%Y-%U-%d")" > .circleci-daily
- restore_cache: - restore_cache:
{% raw %} {% raw %}
keys: keys:
- tp-nix-{{ checksum ".circleci-daily" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }} - tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
{% endraw %} {% endraw %}
- run: - run:
command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only command: ./build_tools/setup_helpers/build_third_party.sh $PWD --download-only
- save_cache: - save_cache:
{% raw %} {% raw %}
key: tp-nix-{{ checksum ".circleci-daily" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }} key: tp-nix-{{ checksum ".cachekey" }}-{{ checksum "./build_tools/setup_helpers/build_third_party.sh" }}-{{ checksum "./build_tools/setup_helpers/build_third_party_helper.sh" }}
{% endraw %} {% endraw %}
paths: paths:
- third_party/tmp - third_party/tmp
...@@ -376,21 +379,18 @@ jobs: ...@@ -376,21 +379,18 @@ jobs:
- checkout - checkout
- attach_workspace: - attach_workspace:
at: third_party at: third_party
- run: - generate_cache_key
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache: - restore_cache:
{% raw %} {% raw %}
keys: keys:
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
{% endraw %} {% endraw %}
- run: - run:
name: Setup name: Setup
command: .circleci/unittest/linux/scripts/setup_env.sh command: .circleci/unittest/linux/scripts/setup_env.sh
- save_cache: - save_cache:
{% raw %} {% raw %}
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
{% endraw %} {% endraw %}
paths: paths:
- conda - conda
...@@ -419,21 +419,18 @@ jobs: ...@@ -419,21 +419,18 @@ jobs:
- checkout - checkout
- attach_workspace: - attach_workspace:
at: third_party at: third_party
- run: - generate_cache_key
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache: - restore_cache:
{% raw %} {% raw %}
keys: keys:
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
{% endraw %} {% endraw %}
- run: - run:
name: Setup name: Setup
command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh command: docker run -t --gpus all -v $PWD:$PWD -w $PWD "${image_name}" .circleci/unittest/linux/scripts/setup_env.sh
- save_cache: - save_cache:
{% raw %} {% raw %}
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
{% endraw %} {% endraw %}
paths: paths:
- conda - conda
...@@ -457,21 +454,18 @@ jobs: ...@@ -457,21 +454,18 @@ jobs:
name: windows-cpu name: windows-cpu
steps: steps:
- checkout - checkout
- run: - generate_cache_key
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache: - restore_cache:
{% raw %} {% raw %}
keys: keys:
- env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - env-v3-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
{% endraw %} {% endraw %}
- run: - run:
name: Setup name: Setup
command: .circleci/unittest/windows/scripts/setup_env.sh command: .circleci/unittest/windows/scripts/setup_env.sh
- save_cache: - save_cache:
{% raw %} {% raw %}
key: env-v2-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} key: env-v3-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
{% endraw %} {% endraw %}
paths: paths:
- conda - conda
...@@ -496,21 +490,18 @@ jobs: ...@@ -496,21 +490,18 @@ jobs:
CUDA_VERSION: "10.1" CUDA_VERSION: "10.1"
steps: steps:
- checkout - checkout
- run: - generate_cache_key
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache: - restore_cache:
{% raw %} {% raw %}
keys: keys:
- env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
{% endraw %} {% endraw %}
- run: - run:
name: Setup name: Setup
command: .circleci/unittest/windows/scripts/setup_env.sh command: .circleci/unittest/windows/scripts/setup_env.sh
- save_cache: - save_cache:
{% raw %} {% raw %}
key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} key: env-v1-windows-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/windows/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
{% endraw %} {% endraw %}
paths: paths:
- conda - conda
...@@ -534,21 +525,18 @@ jobs: ...@@ -534,21 +525,18 @@ jobs:
resource_class: medium resource_class: medium
steps: steps:
- checkout - checkout
- run: - generate_cache_key
name: Generate cache key
# This will refresh cache on Sundays, nightly build should generate new cache.
command: echo "$(date +"%Y-%U")" > .circleci-weekly
- restore_cache: - restore_cache:
{% raw %} {% raw %}
keys: keys:
- env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} - env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
{% endraw %} {% endraw %}
- run: - run:
name: Setup name: Setup
command: .circleci/unittest/linux/scripts/setup_env.sh command: .circleci/unittest/linux/scripts/setup_env.sh
- save_cache: - save_cache:
{% raw %} {% raw %}
key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".circleci-weekly" }} key: env-v2-linux-{{ arch }}-py<< parameters.python_version >>-{{ checksum ".circleci/unittest/linux/scripts/environment.yml" }}-{{ checksum ".cachekey" }}
{% endraw %} {% endraw %}
paths: paths:
- conda - conda
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment