integration-test-openpai-linux.yml

trigger: none
pr: none
schedules:
- cron: 0 16 * * *
  branches:
    include: [ master ]

# variables set on VSO: (mostly for security concern)
#   pai_user
#   pai_token
#   manager_ip
#   docker_hub_password

jobs:
- job: pai
  pool: NNI CI PAI LINUX CLI
  timeoutInMinutes: 120

  steps:
  - script: |
      export NNI_RELEASE=999.$(date -u +%Y%m%d%H%M%S)
      echo "##vso[task.setvariable variable=PATH]${PATH}:${HOME}/.local/bin"
      echo "##vso[task.setvariable variable=NNI_RELEASE]${NNI_RELEASE}"

      echo "Working directory: ${PWD}"
      echo "NNI version: ${NNI_RELEASE}"
      echo "Build docker image: $(build_docker_image)"

      python3 -m pip install --upgrade pip setuptools
    displayName: Prepare

  - script: |
      set -e
      python3 setup.py build_ts
      python3 setup.py bdist_wheel -p manylinux1_x86_64
      python3 -m pip install dist/nni-${NNI_RELEASE}-py3-none-manylinux1_x86_64.whl[SMAC,BOHB]
    displayName: Build and install NNI

  - script: |
      set -e
      cd examples/tuners/customized_tuner
      python3 setup.py develop --user
      nnictl algo register --meta meta_file.yml
    displayName: Install customized tuner

  - script: |
      set -e
      docker login -u nnidev -p $(docker_hub_password)
      echo '## Build docker image ##'
      docker build --build-arg NNI_RELEASE=${NNI_RELEASE} -t nnidev/nni-nightly .
      echo '## Upload docker image ##'
      docker push nnidev/nni-nightly
    condition: eq(variables['build_docker_image'], 'true')
    displayName: Build and upload docker image

  - script: |
      set -e
      cd test
      python3 training_service/nnitest/generate_ts_config.py \
          --ts pai \
          --pai_reuse false \
          --pai_host https://ne.openpai.org \
          --pai_user $(pai_user) \
          --nni_docker_image nnidev/nni-nightly \
          --pai_storage_config_name confignfs-data \
          --pai_token $(pai_token) \
          --nni_manager_nfs_mount_path $(nni_manager_nfs_mount_path) \
          --container_nfs_mount_path $(container_nfs_mount_path) \
          --nni_manager_ip $(manager_ip) \
          --vc nni \
          --debug true
      python3 training_service/nnitest/run_tests.py --config training_service/config/integration_tests.yml --ts pai
    displayName: Integration test

  - script: |
      set -e
      cd test
      python3 training_service/nnitest/generate_ts_config.py \
          --ts pai \
          --pai_reuse true \
          --pai_host https://ne.openpai.org \
          --pai_user $(pai_user) \
          --nni_docker_image nnidev/nni-nightly \
          --pai_storage_config_name confignfs-data \
          --pai_token $(pai_token) \
          --nni_manager_nfs_mount_path $(nni_manager_nfs_mount_path) \
          --container_nfs_mount_path $(container_nfs_mount_path) \
          --nni_manager_ip $(manager_ip) \
          --vc nni
      python3 training_service/nnitest/run_tests.py --config training_service/config/integration_tests.yml --ts pai
    displayName: Integration test (reuse mode)