integration-test-remote-l2l.yml

trigger: none
pr: none
schedules:
- cron: 0 16 * * *
  branches:
    include: [ master ]

variables:
  worker: remote_nni-ci-gpu-03

# variables set on VSO: (for security concern)
#   manager_ip
#   worker_ip
#   password_in_docker

jobs:
- job: remote_linux2linux
  pool: NNI CI REMOTE CLI
  timeoutInMinutes: 120

  steps:
  - script: |
      export NNI_RELEASE=999.$(date -u +%Y%m%d%H%M%S)
      echo "##vso[task.setvariable variable=PATH]${PATH}:${HOME}/.local/bin"
      echo "##vso[task.setvariable variable=NNI_RELEASE]${NNI_RELEASE}"

      echo "Working directory: ${PWD}"
      echo "NNI version: ${NNI_RELEASE}"

      python3 -m pip install -U --upgrade pip setuptools
    displayName: Prepare

  - script: |
      set -e
      python3 setup.py build_ts
      python3 setup.py bdist_wheel -p manylinux1_x86_64
      python3 -m pip install dist/nni-${NNI_RELEASE}-py3-none-manylinux1_x86_64.whl
    displayName: Install NNI

  - script: |
      set -e
      sudo apt-get install swig -y
      nnictl package install --name=SMAC
      nnictl package install --name=BOHB
    displayName: Install extra tuners

  - script: |
      set -e
      cd examples/tuners/customized_tuner
      python3 setup.py develop --user
      nnictl package install .
    displayName: Install customized tuner

  - task: CopyFilesOverSSH@0
    inputs:
      sshEndpoint: $(worker)
      sourceFolder: dist
      targetFolder: /tmp/nnitest/$(Build.BuildId)/dist
      overwrite: true
    displayName: Copy wheel to remote machine
    timeoutInMinutes: 10

  - task: CopyFilesOverSSH@0
    inputs:
      sshEndpoint: $(worker)
      sourceFolder: test
      targetFolder: /tmp/nnitest/$(Build.BuildId)/test
      overwrite: true
    displayName: Copy test scripts to remote machine
    timeoutInMinutes: 10

  - task: SSH@0
    inputs:
      sshEndpoint: $(worker)
      runOptions: commands
      commands: |
        python3 /tmp/nnitest/$(Build.BuildId)/test/nni_test/nnitest/remote_docker.py --mode start --name $(Build.BuildId) --image nni/nni
        echo "##vso[task.setvariable variable=docker_port]$(cat /tmp/nnitest/$(Build.BuildId)/port)"
    displayName: Start docker

  - script: |
      cd test
      python3 nni_test/nnitest/generate_ts_config.py \
          --ts remote \
          --remote_reuse false \
          --remote_user nni \
          --remote_host $(worker_ip) \
          --remote_port $(docker_port) \
          --remote_pwd $(password_in_docker) \
          --nni_manager_ip $(manager_ip)
      python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote
    displayName: Integration test

  - script: |
      cd test
      python3 nni_test/nnitest/generate_ts_config.py \
          --ts remote \
          --remote_reuse true \
          --remote_user nni \
          --remote_host $(worker_ip) \
          --remote_port $(docker_port) \
          --remote_pwd $(password_in_docker) \
          --nni_manager_ip $(manager_ip)
      python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote
    displayName: Integration test (reuse mode)

  - task: SSH@0
    inputs:
      sshEndpoint: $(worker)
      runOptions: commands
      commands: python3 /tmp/nnitest/$(Build.BuildId)/test/nni_test/nnitest/remote_docker.py --mode stop --name $(Build.BuildId)
    displayName: Stop docker