Refactor integration test (step 8) - pipeline updates (windows part) (#4899)

pipelines/integration-test-local-windows.yml

+trigger: none
+pr: none
+schedules:
+- cron: 0 16 * * *
+  branches:
+    include: [ master ]
+
+jobs:
+- job: windows
+  pool: nni-it-windows
+  timeoutInMinutes: 120
+
+  steps:
+
+  - template: templates/install-dependencies.yml
+    parameters:
+      platform: windows
+      python_venv: noop
+
+  - template: templates/install-nni.yml
+    parameters:
+      user: false
+
+  - template: templates/install-customized-tuner.yml
+
+  - powershell: |
+      python test/vso_tools/ssl_patch.py
+    displayName: SSL patch
+
+  - powershell: |
+      cd test
+      python -m pytest ut
+      echo "TODO: TypeScript UT"
+    displayName: Unit test
+    continueOnError: true
+
+  # TODO: the agent has no GPU driver
+  # We can't install it on-the-fly because we can't elevate the permission here.
+  - powershell: |
+      cd test
+      python training_service/nnitest/run_tests.py --config training_service/config/integration_tests.yml --ts local `
+        --exclude mnist-pytorch-local-gpu
+    displayName: Integration test
+
+  - template: templates/save-crashed-info.yml

pipelines/integration-test-remote-l2w.yml

@@ -66,14 +66,14 @@ jobs:
   - script: |
       set -e
       cd test
-      python3 nni_test/nnitest/generate_ts_config.py \
+      python3 training_service/nnitest/generate_ts_config.py \
           --ts remote \
           --remote_user AzureUser \
           --remote_host $(worker_ip) \
           --remote_port $(worker_port) \
           --remote_pwd $(worker_password) \
           --nni_manager_ip $(manager_ip)
-      python3 nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote
+      python3 training_service/nnitest/run_tests.py --config training_service/config/integration_tests.yml --ts remote
     displayName: Integration test
 
   - task: SSH@0

pipelines/integration-test-remote-w2l.yml

@@ -53,7 +53,7 @@ jobs:
 
   - powershell: |
       cd test
-      python nni_test/nnitest/generate_ts_config.py `
+      python training_service/nnitest/generate_ts_config.py `
           --ts remote `
           --remote_reuse false `
           --remote_user nni `
@@ -61,8 +61,8 @@ jobs:
           --remote_port $(docker_port) `
           --remote_pwd $(password_in_docker) `
           --nni_manager_ip $(manager_ip)
-      Get-Content config/training_service.yml
-      python nni_test/nnitest/run_tests.py --config config/integration_tests.yml --ts remote --exclude cifar10
+      Get-Content training_service/config/training_service.yml
+      python training_service/nnitest/run_tests.py --config training_service/config/integration_tests.yml --ts remote --exclude cifar10
     displayName: Integration test
 
   - task: SSH@0

pipelines/integration-test-remote-w2w.yml

+trigger: none
+pr: none
+# schedules:
+# - cron: 0 16 * * *
+#   branches:
+#     include: [ master ]
+
+variables:
+  ssh_user: NNIUser
+  ssh_password: P@ssW0rD!
+
+jobs:
+- job: remote_windows2windows
+  pool: nni-it-windows
+  timeoutInMinutes: 60
+
+  steps:
+  # FIXME: GPU is not supported yet.
+  - template: templates/install-dependencies.yml
+    parameters:
+      platform: windows
+      python_venv: noop
+
+  - template: templates/install-nni.yml
+    parameters:
+      user: false
+
+  - template: templates/install-customized-tuner.yml
+
+  - powershell: |
+      python test/vso_tools/ssl_patch.py
+    displayName: SSL patch
+
+  # Connects to itself.
+  # See username and password in test/vso_tools/build_vm
+  - powershell: |
+      cd test
+      python training_service/nnitest/generate_ts_config.py `
+          --ts remote `
+          --remote_reuse false `
+          --remote_user $(ssh_user) `
+          --remote_host localhost `
+          --remote_port 22 `
+          --remote_pwd "$(ssh_password)" `
+          --nni_manager_ip localhost
+      Get-Content training_service/config/training_service.yml
+      python training_service/nnitest/run_tests.py --config training_service/config/integration_tests.yml --ts remote
+    displayName: Integration test
+
+  - template: templates/save-crashed-info.yml
+    parameters:
+      remote: true

test/vso_tools/ssl_patch.py

+"""Fix the issue: [SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: unable to get local issuer certificate (_ssl.c:1129).
+
+I think it's caused by some incorrect certificates injected by WinRM,
+but I guess disabling the check is a simpler way.
+"""
+
+import ssl
+from pathlib import Path
+
+ssl_file_path = ssl.__file__
+print('SSL file path:', ssl_file_path)
+
+# https://stackoverflow.com/questions/36600583/python-3-urllib-ignore-ssl-certificate-verification
+old_line = '_create_default_https_context = create_default_context'
+new_line = '_create_default_https_context = _create_unverified_context'
+
+new_ssl_file_content = Path(ssl_file_path).read_text().replace(old_line, new_line)
+Path(ssl_file_path).write_text(new_ssl_file_content)