Refactor integration test (step 6) - pipeline templates (#4897)

pipelines/templates/config-version.yml

 parameters:
 - name: platform
   type: string
+- name: python_env
+  type: string
+  default: default
+  values:
+  - default
+  - noop
+  - venv
 
 steps:
-- task: UsePythonVersion@0
-  inputs:
-    versionSpec: 3.9
-  displayName: (latest) Configure Python version
-  condition: and(succeeded(), not(contains('${{ parameters.platform }}', 'legacy')))
+
+# UsePythonVersion task only works when the specific Python version is already installed.
+
+# The following is for linux.
+# Reference: https://dev.to/akaszynski/create-an-azure-self-hosted-agent-without-going-insane-173g
+# We only need Python 3.7 and 3.9 for now.
+# --system-site-packages is required to make packages installed with --user visible to virtualenv.
+- ${{ if contains(parameters.platform, 'legacy') }}:
+    script: |
+      set -e
+      mkdir -p $(Agent.ToolsDirectory)/Python
+      cd $(Agent.ToolsDirectory)/Python
+      PY37_VER=$(python3.7 -c "import sys; print('.'.join([f'{val}' for val in sys.version_info[:3]]))")
+      mkdir $PY37_VER
+      ln -s $PY37_VER 3.7
+      cd $PY37_VER
+      python3.7 -m venv x64 --system-site-packages
+      touch x64.complete
+    displayName: Create Python 3.7 venv
+  ${{ else }}:
+    script: |
+      set -e
+      mkdir -p $(Agent.ToolsDirectory)/Python
+      cd $(Agent.ToolsDirectory)/Python
+      PY39_VER=$(python3.9 -c "import sys; print('.'.join([f'{val}' for val in sys.version_info[:3]]))")
+      mkdir $PY39_VER
+      ln -s $PY39_VER 3.9
+      cd $PY39_VER
+      python3.9 -m venv x64 --system-site-packages
+      touch x64.complete
+    displayName: Create Python 3.9 venv
+  condition: and(succeeded(), eq('${{ parameters.python_env }}', 'venv'))
 
 - task: UsePythonVersion@0
   inputs:
-    versionSpec: 3.7
-  displayName: (legacy) Configure Python version
-  condition: and(succeeded(), contains('${{ parameters.platform }}', 'legacy'))
+    ${{ if contains(parameters.platform, 'legacy') }}:
+      versionSpec: 3.7
+    ${{ else }}:
+      versionSpec: 3.9
+  displayName: Configure Python version
+  condition: and(succeeded(), ne('${{ parameters.python_env }}', 'noop'))
 
 - task: NodeTool@0
   inputs:

pipelines/templates/install-customized-tuner.yml

+steps:
+
+- ${{ if contains(variables['Agent.OS'], 'Windows') }}:
+    powershell: |
+      cd examples/tuners/customized_tuner
+      python setup.py develop --user
+      nnictl algo register --meta meta_file.yml
+  ${{ else }}:
+    script: |
+      set -e
+      cd examples/tuners/customized_tuner
+      python setup.py develop
+      nnictl algo register --meta meta_file.yml
+  displayName: Install customized tuner

pipelines/templates/install-dependencies-aml.yml

+steps:
+
+# Need to have a service principal (SP) to login to Azure services (e.g., AzureML).
+# Refer to external account section in OneNote for how to generate / renew the authorization.
+# According to docs, the secrets need to be refreshed at least once per year.
+# After the tokens are updated, pipeline secret varibles should be updated correspondingly.
+
+- script: |
+    az login --service-principal -u $(client_id) -p $(client_secret) --tenant $(tenant_id)
+  displayName: Login to Azure
+
+# It's tricky to install Azure SDKs.
+- script: |
+    set -e
+    # Separate installation in two steps for easy debugging
+    python -m pip install azureml-core
+    python -m pip install azure-cli-core
+    # Azure CLI and AzureML have conflicted requirements for msal-extensions.
+    python -m pip install msal-extensions==0.3.1
+  displayName: Install AzureML requirements

pipelines/templates/install-dependencies.yml

 parameters:
 - name: platform
   type: string
+- name: python_env
+  type: string
+  default: default
+  values:
+  - default
+  - noop
+  - venv
 
 steps:
 - template: config-version.yml
   parameters:
     platform: ${{ parameters.platform }}
+    python_env: ${{ parameters.python_env }}
 
 - script: |
     set -e
@@ -27,6 +35,7 @@ steps:
     set -e
     azcopy copy 'https://nni.blob.core.windows.net/cache/dependencies-${{ parameters.platform }}.zip' dependencies.zip
     python test/vso_tools/unpack_dependencies.py dependencies.zip
+    rm dependencies.zip
   displayName: (POSIX) Download cache
   condition: and(succeeded(), not(contains('${{ parameters.platform }}', 'windows')))
   continueOnError: true
@@ -34,6 +43,7 @@ steps:
 - powershell: |
     azcopy copy 'https://nni.blob.core.windows.net/cache/dependencies-${{ parameters.platform }}.zip' dependencies.zip
     python test/vso_tools/unpack_dependencies.py dependencies.zip
+    Remove-Item dependencies.zip
   displayName: (Windows) Download cache
   condition: and(succeeded(), contains('${{ parameters.platform }}', 'windows'))
   continueOnError: true
@@ -43,8 +53,12 @@ steps:
   displayName: (legacy) Activate legacy dependencies
   condition: and(succeeded(), contains('${{ parameters.platform }}', 'legacy'))
 
-- script: |
-    mv dependencies/recommended_gpu.txt dependencies/recommended.txt
+- ${{ if contains(parameters.platform, 'windows') }}:
+    powershell:
+      Move-Item -Force -Path dependencies/recommended_gpu.txt -Destination dependencies/recommended.txt
+  ${{ else }}:
+    script: |
+      mv dependencies/recommended_gpu.txt dependencies/recommended.txt
   displayName: (GPU) Activate CUDA dependencies
   condition: and(succeeded(), contains('${{ parameters.platform }}', 'gpu'))
 
@@ -62,7 +76,7 @@ steps:
 - script: |
     ls -al ${HOME}/.local/bin
     echo "##vso[task.setvariable variable=PATH]${HOME}/.local/bin:${PATH}"
-  displayName: Add .local/bin to PATH
+  displayName: (POSIX) Add local bin to PATH
   condition: and(succeeded(), not(contains('${{ parameters.platform }}', 'windows')))
 
 # TODO: Delete this after upgrading to PyTorch 1.11.
@@ -71,5 +85,7 @@ steps:
   displayName: Torch utils tensorboard interim patch
 
 - script: |
-    pip list
+    python --version
+    python -m pip --version
+    python -m pip list
   displayName: List pip dependencies

pipelines/templates/install-nni.yml

@@ -2,13 +2,30 @@ parameters:
 - name: user
   type: boolean
   default: true
+- name: wheel
+  type: boolean
+  default: false
+- name: extra_dep
+  type: string
+  default: ""
 
 steps:
 
-- ${{ if eq(parameters.user, true) }}:
+- ${{ if eq(parameters.wheel, true) }}:
+    script: |
+      echo "Install NNI (wheel)"
+      export NNI_RELEASE=999.$(date -u +%Y%m%d%H%M%S)
+      echo "##vso[task.setvariable variable=NNI_RELEASE]${NNI_RELEASE}"
+      echo "Working directory: $(pwd)"
+      echo "Extra dependencies: ${{ parameters.extra_dep }}"
+      echo "NNI release version: ${NNI_RELEASE}"
+      python test/vso_tools/install_nni.py ${NNI_RELEASE} ${{ parameters.extra_dep }}
+  ${{ elseif eq(parameters.user, true) }}:
     script: |
+      echo "Install NNI (user)"
       python setup.py develop
   ${{ else }}:
     script: |
+      echo "Install NNI (no user)"
       python setup.py develop --no-user
   displayName: Install NNI

pipelines/templates/save-crashed-info.yml

+# Upload crashed experiments to artifact,
+# so that further offline investigations are possible.
+
+parameters:
+- name: remote
+  type: boolean
+  default: false
+
+steps:
+
+- script: |
+    set -e
+    export EXPERIMENT_DIR=$(readlink -f ~/nni-experiments/_latest)
+    echo "Latest experiment directory: ${EXPERIMENT_DIR}"
+    echo "##vso[task.setvariable variable=experiment_dir]${EXPERIMENT_DIR}"
+  condition: and(failed(), not(contains(variables['Agent.OS'], 'Windows')))
+  displayName: (failed) (POSIX) Latest experiment directory
+
+- script: |
+    set -e
+    export EXPERIMENT_ID=$(echo ${EXPERIMENT_DIR} | sed -e 's/\/.*\///g')
+    sudo docker cp $(Build.BuildId):/tmp/nni-experiments/${EXPERIMENT_ID} ${EXPERIMENT_DIR}/remote && echo "Copy successful" || echo "Copy failed"
+  condition: and(variables['experiment_dir'], ${{ parameters.remote }}, not(contains(variables['Agent.OS'], 'Windows')))
+  displayName: (failed) (POSIX) Harvest remote trial logs
+
+- powershell: |
+    $latestDir = (gci ~/nni-experiments -exclude _latest | ? { $_.PSIsContainer } | sort CreationTime)[-1]
+    echo "Latest experiment directory: $latestDir"
+    echo "##vso[task.setvariable variable=experiment_dir]$latestDir"
+  condition: and(failed(), contains(variables['Agent.OS'], 'Windows'))
+  displayName: (failed) (Windows) Latest experiment directory
+
+- powershell: |
+    $latestDir = Get-Item $(experiment_dir)
+    $experimentId = $latestDir.name
+    $remotePath = "C:\Users\nniuser\AppData\Local\Temp\nni-experiments\${experimentId}"
+    $destPath = "${latestDir}\remote"
+
+    if (Test-Path $remotePath) {
+      Write-Host "Copying $remotePath to $destPath"
+      Copy-Item $remotePath -Destination $destPath -Recurse
+    }
+    else {
+      Write-host "$remotePath doesn't exist"
+    }
+  condition: and(variables['experiment_dir'], ${{ parameters.remote }}, contains(variables['Agent.OS'], 'Windows'))
+  displayName: (failed) (Windows) Harvest remote trial logs
+
+- publish: $(experiment_dir)
+  artifact: experiment
+  condition: variables['experiment_dir']
+  displayName: (failed) Upload experiment artifact

pipelines/templates/setup-non-ms-hosted-agent.yml

-steps:
-
-# OS from VMSS is very clean... Need to install basic utilities.
-
-# Build essentials are required.
-
-- script: |
-    set -e
-    sudo apt update
-    sudo apt install -y build-essential cmake
-  displayName: Install build essential
-
-
-# Install azcopy for cache download.
-# https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-v10#use-azcopy-in-a-script
-
-- script: |
-    set -e
-    mkdir -p tmp
-    cd tmp
-    wget -O azcopy_v10.tar.gz https://aka.ms/downloadazcopy-v10-linux && tar -xf azcopy_v10.tar.gz --strip-components=1
-    sudo cp ./azcopy /usr/bin/
-    sudo chmod +x /usr/bin/azcopy
-  displayName: Setup azcopy
-
-# VM with GPU needs to install drivers. Reference:
-# https://docs.microsoft.com/en-us/azure/virtual-machines/linux/n-series-driver-setup
-# https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html
-# https://linuxhint.com/install-cuda-ubuntu/
-
-- script: |
-    lspci | grep -i NVIDIA
-  displayName: GPU status verification
-
-- script: |
-    set -e
-    sudo apt install linux-headers-$(uname -r) -y
-    sudo wget -O /etc/apt/preferences.d/cuda-repository-pin-600 https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/cuda-ubuntu2004.pin
-    sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub
-    sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /"
-    sudo apt update
-    sudo apt install -y cuda-drivers
-  displayName: Install CUDA
-
-# Technically we need a reboot here, but looks like it also works without reboot.
-
-- script: |
-    nvidia-smi
-  displayName: nvidia-smi verification
-
-
-# UsePythonVersion task only works when the specific Python version is already installed.
-
-# The following is for linux.
-# Reference: https://dev.to/akaszynski/create-an-azure-self-hosted-agent-without-going-insane-173g
-# We only need Python 3.7 and 3.9 for now.
-# --system-site-packages is required to make packages installed with --user visible to virtualenv.
-- script: |
-    set -e
-    sudo add-apt-repository ppa:deadsnakes/ppa
-    sudo apt install -y python3.7-dev python3.7-venv python3.9-dev python3.9-venv
-    mkdir $(Agent.ToolsDirectory)/Python
-  displayName: Download Python
-
-- script: |
-    set -e
-    cd $(Agent.ToolsDirectory)/Python
-    PY37_VER=$(python3.7 -c "import sys; print('.'.join([f'{val}' for val in sys.version_info[:3]]))")
-    mkdir $PY37_VER
-    ln -s $PY37_VER 3.7
-    cd $PY37_VER
-    python3.7 -m venv x64 --system-site-packages
-    touch x64.complete
-  displayName: Configure Python 3.7
-
-- script: |
-    set -e
-    cd $(Agent.ToolsDirectory)/Python
-    PY39_VER=$(python3.9 -c "import sys; print('.'.join([f'{val}' for val in sys.version_info[:3]]))")
-    mkdir $PY39_VER
-    ln -s $PY39_VER 3.9
-    cd $PY39_VER
-    python3.9 -m venv x64 --system-site-packages
-    touch x64.complete
-  displayName: Configure Python 3.9