jobs:
- job: 'integration_test_remote_linux_to_linux'
  timeoutInMinutes: 120

  steps:
  - script: python3 -m pip install --upgrade pip setuptools --user
    displayName: 'Install python tools'
  - script: |
      set -e
      cd deployment/pypi
      echo 'building prerelease package...'
      make build
      ls $(Build.SourcesDirectory)/deployment/pypi/dist/
    displayName: 'build nni bdsit_wheel'
  - script: |
      source install.sh
    displayName: 'Install nni toolkit via source code'
  - script: |
      set -e
      sudo apt-get install swig -y
      PATH=$HOME/.local/bin:$PATH nnictl package install --name=SMAC
      PATH=$HOME/.local/bin:$PATH nnictl package install --name=BOHB
    displayName: 'Install dependencies for integration tests in remote mode'
  - script: |
      set -e
      cd examples/tuners/customized_tuner
      python3 setup.py develop --user
      PATH=$HOME/.local/bin:$PATH nnictl package install ./
    displayName: 'Install demotuner for customized_tuner test'
  - task: CopyFilesOverSSH@0
    inputs:
      sshEndpoint: $(end_point)
      sourceFolder: deployment/pypi/dist/
      targetFolder: /tmp/nnitest/$(Build.BuildId)/dist
      overwrite: true
    displayName: 'Copy dist files to remote machine'
    timeoutInMinutes: 10
  - task: CopyFilesOverSSH@0
    inputs:
      sshEndpoint: $(end_point)
      sourceFolder: test
      targetFolder: /tmp/nnitest/$(Build.BuildId)/test
      overwrite: true
    displayName: 'Copy test files to remote machine'
    timeoutInMinutes: 10
  - task: SSH@0
    inputs:
      sshEndpoint: $(end_point)
      runOptions: commands
      commands: python3 /tmp/nnitest/$(Build.BuildId)/test/nni_test/nnitest/remote_docker.py --mode start --name $(Build.BuildId) --image nni/nni
    displayName: 'Start docker'
  - task: DownloadSecureFile@1
    inputs:
      secureFile: $(remote_private_key)
  - script: |
      set -e
      cp $(Agent.TempDirectory)/$(remote_private_key) test/id_rsa
      chmod 600 test/id_rsa
      scp -P $(remote_port) -i test/id_rsa $(remote_user)@$(remote_host):/tmp/nnitest/$(Build.BuildId)/port test/port
      cat test/port
    displayName: 'Get docker port'
  - script: |
      set -e
      cd test
      python3 nni_test/nnitest/generate_ts_config.py --ts remote --remote_reuse $(remote_reuse) --remote_user $(docker_user) --remote_host $(remote_host) \
      --remote_port $(cat port) --remote_pwd $(docker_pwd) --nni_manager_ip $(nni_manager_ip)
      cat config/training_service.yml
      PATH=$HOME/.local/bin:$PATH python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote
    displayName: 'integration test'
  - task: SSH@0
    inputs:
      sshEndpoint: $(end_point)
      runOptions: commands
      commands: python3 /tmp/nnitest/$(Build.BuildId)/test/nni_test/nnitest/remote_docker.py --mode stop --name $(Build.BuildId)
    displayName: 'Stop docker'