CI/CD - Add integration tests for Ansible playbooks (#82)

* Add integration tests for Ansible playbooks
* Add `gpu_vendor` var to bypass gpu mount

.azure-pipelines/ansible-integration-test.yml

+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+trigger:
+  - main
+
+pool:
+  name: SuperBench CI
+  vmImage: ubuntu-latest
+
+container:
+  image: nvcr.io/nvidia/pytorch:20.12-py3
+
+steps:
+  - task: DownloadSecureFile@1
+    name: vagrantKeyFile
+    displayName: Download key file
+    inputs:
+      secureFile: vagrant.key
+  - script: |
+      echo "##vso[task.prependpath]$HOME/.local/bin"
+    displayName: Export path
+  - script: |
+      python3 -m pip install .[test]
+      make postinstall
+    displayName: Install Ansible dependencies
+  - script: |
+      chmod 400 $(vagrantKeyFile.secureFilePath)
+      sed -i "s\vagrant.key\$(vagrantKeyFile.secureFilePath)\g" tests/ansible/hosts.ini
+      export ANSIBLE_HOST_KEY_CHECKING=False
+      bash tests/ansible/test.sh
+    displayName: Run integration tests for Ansible playbooks
+    timeoutInMinutes: 10

superbench/runner/playbooks/deploy.yaml

@@ -64,7 +64,9 @@
       shell: |
         docker rm --force {{ container }} ||: && \
         docker run -itd --name={{ container }} \
-          --privileged --net=host --ipc=host --gpus=all \
+          --privileged --net=host --ipc=host \
+          {{ '--gpus=all' if gpu_vendor == 'nvidia' }} \
+          {{ '--security-opt seccomp=unconfined --group-add video' if gpu_vendor == 'amd' }} \
           -w /root -v {{ workspace }}:/root -v /mnt:/mnt \
           {{ docker_image }} bash && \
         docker exec {{ container }} bash -c \

superbench/runner/runner.py

@@ -65,6 +65,7 @@ def deploy(self):    # pragma: no cover
             'ssh_port': random.randint(1 << 14, (1 << 15) - 1),
             'output_dir': self._output_dir,
             'docker_image': self._docker_config.image,
+            'gpu_vendor': 'nvidia',
         }
         if bool(self._docker_config.username) and bool(self._docker_config.password):
             extravars.update(

tests/ansible/Vagrantfile

+# -*- mode: ruby -*-
+
+Vagrant.configure("2") do |config|
+    (0..2).each do |id|
+        config.vm.define "node#{id}" do |node|
+            node.vm.box = "ubuntu/focal64"
+            node.vm.box_check_update = false
+            node.vm.hostname = "node#{id}"
+            node.vm.network "private_network",
+                ip: "192.168.0.#{100+id}"
+            node.vm.network "forwarded_port",
+                id: "ssh", guest: 22, host: 22000+id, host_ip: "127.0.0.1"
+            node.vm.provision "docker",
+                images: ["superbench/superbench"]
+            node.vm.provision "shell",
+                inline: "cat /vagrant/ed25519.pub >> /home/vagrant/.ssh/authorized_keys"
+            node.vm.provider "virtualbox" do |vb|
+                vb.name = "node#{id}"
+                vb.memory = "1024"
+            end
+        end
+    end
+end

tests/ansible/hosts.ini

+[vagrant]
+192.168.0.[100:102] ansible_user=vagrant ansible_ssh_private_key_file=vagrant.key

tests/ansible/test.sh

+#!/bin/bash
+
+test_dir="$(dirname $0)"
+test_cases=(
+    test_deploy
+    test_check_env
+)
+
+err_num=0
+for test_case in ${test_cases[@]}; do
+    ansible-playbook -i "${test_dir}/hosts.ini" "${test_dir}/tests/${test_case}.yaml"
+    rc=$?
+    if [ ${rc} -eq 0 ]; then
+        echo "tests ${test_case} PASSED"
+    else
+        echo "tests ${test_case} FAILED"
+        err_num=$((err_num+1))
+    fi
+done
+
+exit ${err_num}

tests/ansible/tests/test_check_env.yaml

+- name: Context Preparation
+  hosts: localhost
+  connection: localhost
+  gather_facts: false
+  tasks:
+    - name: Creating Config File
+      copy:
+        src: ../../../superbench/config/default.yaml
+        dest: /tmp/test_ansible/sb.config.yaml
+
+- name: Test Playbook check_env.yaml
+  import_playbook: ../../../superbench/runner/playbooks/check_env.yaml
+  vars:
+    output_dir: /tmp/test_ansible

tests/ansible/tests/test_deploy.yaml

+- name: Context Preparation
+  hosts: localhost
+  connection: localhost
+  gather_facts: false
+  tasks:
+    - name: Creating Directory
+      file:
+        path: /tmp/test_ansible
+        state: directory
+        mode: 0755
+
+- name: Test Playbook deploy.yaml
+  import_playbook: ../../../superbench/runner/playbooks/deploy.yaml
+  vars:
+    ssh_port: 12345
+    output_dir: /tmp/test_ansible
+    docker_image: superbench/superbench
+    gpu_vendor: none