integration-test-remote-l2l.yml 3.62 KB
Newer Older
liuzhe-lz's avatar
liuzhe-lz committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
trigger: none
pr: none
schedules:
- cron: 0 16 * * *
  branches:
    include: [ master ]

variables:
  worker: remote_nni-ci-gpu-03

# variables set on VSO: (for security concern)
#   manager_ip
#   worker_ip
#   password_in_docker

jobs:
- job: remote_linux2linux
  pool: NNI CI REMOTE CLI
  timeoutInMinutes: 120

  steps:
  - script: |
      export NNI_RELEASE=999.$(date -u +%Y%m%d%H%M%S)
      echo "##vso[task.setvariable variable=PATH]${PATH}:${HOME}/.local/bin"
      echo "##vso[task.setvariable variable=NNI_RELEASE]${NNI_RELEASE}"

      echo "Working directory: ${PWD}"
      echo "NNI version: ${NNI_RELEASE}"

30
      python3 -m pip install --upgrade pip setuptools
liuzhe-lz's avatar
liuzhe-lz committed
31
32
33
34
    displayName: Prepare

  - script: |
      set -e
35
      python3 test/vso_tools/install_nni.py $(NNI_RELEASE) SMAC,BOHB
liuzhe-lz's avatar
liuzhe-lz committed
36
37
38

      cd examples/tuners/customized_tuner
      python3 setup.py develop --user
39
40
      nnictl algo register --meta meta_file.yml
    displayName: Install NNI
liuzhe-lz's avatar
liuzhe-lz committed
41
42
43
44
45
46
47
48
49
50

  - task: CopyFilesOverSSH@0
    inputs:
      sshEndpoint: $(worker)
      sourceFolder: dist
      targetFolder: /tmp/nnitest/$(Build.BuildId)/dist
      overwrite: true
    displayName: Copy wheel to remote machine
    timeoutInMinutes: 10

51
52
53
54
55
56
57
58
59
  - task: CopyFilesOverSSH@0
    inputs:
      sshEndpoint: $(worker)
      contents: Dockerfile
      targetFolder: /tmp/nnitest/$(Build.BuildId)
      overwrite: true
    displayName: Copy dockerfile to remote machine
    timeoutInMinutes: 10

liuzhe-lz's avatar
liuzhe-lz committed
60
61
62
63
64
65
66
67
68
  - task: CopyFilesOverSSH@0
    inputs:
      sshEndpoint: $(worker)
      sourceFolder: test
      targetFolder: /tmp/nnitest/$(Build.BuildId)/test
      overwrite: true
    displayName: Copy test scripts to remote machine
    timeoutInMinutes: 10

J-shang's avatar
J-shang committed
69
70
71
72
73
74
75
76
77
78
  # Need del later
  - task: CopyFilesOverSSH@0
    inputs:
      sshEndpoint: $(worker)
      contents: interim_vision_patch.py
      targetFolder: /tmp/nnitest/$(Build.BuildId)
      overwrite: true
    displayName: Copy vision patch to remote machine
    timeoutInMinutes: 10

liuzhe-lz's avatar
liuzhe-lz committed
79
80
81
82
  - task: SSH@0
    inputs:
      sshEndpoint: $(worker)
      runOptions: commands
83
84
      commands: python3 /tmp/nnitest/$(Build.BuildId)/test/vso_tools/start_docker.py $(NNI_RELEASE) $(Build.BuildId) $(password_in_docker)
    displayName: Install NNI and run docker on Linux worker
liuzhe-lz's avatar
liuzhe-lz committed
85
86
87
88
89

  - script: |
      cd test
      python3 nni_test/nnitest/generate_ts_config.py \
          --ts remote \
90
          --remote_reuse true \
liuzhe-lz's avatar
liuzhe-lz committed
91
92
93
94
          --remote_user nni \
          --remote_host $(worker_ip) \
          --remote_port $(docker_port) \
          --remote_pwd $(password_in_docker) \
95
96
          --nni_manager_ip $(manager_ip) \
          --azurestoragetoken $(azureblob_token_test) \
97
98
99
100
          --nfs_server $(NFS_IP) \
          --local_mount_point $(LOCAL_MOUNT_POINT) \
          --remote_mount_point $(REMOTE_MOUNT_POINT) \
          --exported_directory $(Exported_Directory)
liuzhe-lz's avatar
liuzhe-lz committed
101
      python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote
102
    displayName: Integration test (reuse mode)
liuzhe-lz's avatar
liuzhe-lz committed
103
104
105
106
107

  - script: |
      cd test
      python3 nni_test/nnitest/generate_ts_config.py \
          --ts remote \
108
          --remote_reuse false \
liuzhe-lz's avatar
liuzhe-lz committed
109
110
111
112
113
114
          --remote_user nni \
          --remote_host $(worker_ip) \
          --remote_port $(docker_port) \
          --remote_pwd $(password_in_docker) \
          --nni_manager_ip $(manager_ip)
      python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote
115
116
    displayName: Integration test

liuzhe-lz's avatar
liuzhe-lz committed
117
118
119
120
121

  - task: SSH@0
    inputs:
      sshEndpoint: $(worker)
      runOptions: commands
122
123
      commands: python3 /tmp/nnitest/$(Build.BuildId)/test/vso_tools/stop_docker.py $(Build.BuildId)
    condition: always()
liuzhe-lz's avatar
liuzhe-lz committed
124
    displayName: Stop docker