pipelines-it-remote.yml

jobs:
- job: 'integration_test_remote'

  steps:
  - script: python3 -m pip install --upgrade pip setuptools --user
    displayName: 'Install python tools'
  - script: |
      cd deployment/pypi
      echo 'building prerelease package...'
      make build
      ls $(Build.SourcesDirectory)/deployment/pypi/dist/
    displayName: 'build nni bdsit_wheel'
  - script: |
      source install.sh
    displayName: 'Install nni toolkit via source code'
  - script: |
      sudo apt-get install swig -y
      PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC
      PATH=$HOME/.local/bin:$PATH nnictl package install --name=BOHB
    displayName: 'Install dependencies for integration tests in remote mode'
  - task: CopyFilesOverSSH@0
    inputs:
      sshEndpoint: $(end_point)
      sourceFolder: deployment/pypi/dist/
      targetFolder: /tmp/nnitest/$(Build.BuildId)/dist
      overwrite: true
    displayName: 'Copy dist files to remote machine'
  - task: CopyFilesOverSSH@0
    inputs:
      sshEndpoint: $(end_point)
      sourceFolder: test
      targetFolder: /tmp/nnitest/$(Build.BuildId)/test
      overwrite: true
    displayName: 'Copy test files to remote machine'
  - task: SSH@0
    inputs:
      sshEndpoint: $(end_point)
      runOptions: commands
      commands: python3 /tmp/nnitest/$(Build.BuildId)/test/remote_docker.py --mode start --name $(Build.BuildId) --image nni/nni
    displayName: 'Start docker'
  - task: DownloadSecureFile@1
    inputs:
      secureFile: $(remote_private_key)
  - script: |
      cp $(Agent.TempDirectory)/$(remote_private_key) test/id_rsa
      chmod 600 test/id_rsa
      scp -P $(remote_port) -i test/id_rsa $(remote_user)@$(remote_host):/tmp/nnitest/$(Build.BuildId)/port test/port
      cat test/port
    displayName: 'Get docker port'
  - script: |
      set -e
      cd test
      python3 generate_ts_config.py --ts remote --remote_user $(docker_user) --remote_host $(remote_host) \
      --remote_port $(cat port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip)
      cat training_service.yml
      PATH=$HOME/.local/bin:$PATH python3 config_test.py --ts remote --exclude cifar10
      PATH=$HOME/.local/bin:$PATH python3 metrics_test.py
    displayName: 'integration test'
  - task: SSH@0
    inputs:
      sshEndpoint: remote_nni-ci-gpu-01
      runOptions: commands
      commands: python3 /tmp/nnitest/$(Build.BuildId)/test/remote_docker.py --mode stop --name $(Build.BuildId)
    displayName: 'Stop docker'