Refactor integration test (step 4) - refactor test file tree structure (#4895)

pipelines/fast-test.yml

@@ -155,7 +155,7 @@ stages:
 
     - script: |
         cd test
-        python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
+        python training_service/nnitest/run_tests.py --config training_service/config/pr_tests.yml
       displayName: Simple integration test
 
   - job: ubuntu_legacy
@@ -171,7 +171,7 @@ stages:
 
     - script: |
         cd test
-        python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
+        python training_service/nnitest/run_tests.py --config training_service/config/pr_tests.yml
       displayName: Simple integration test
 
     - script: |
@@ -210,7 +210,7 @@ stages:
 
     - script: |
         cd test
-        python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
+        python training_service/nnitest/run_tests.py --config training_service/config/pr_tests.yml
       displayName: Simple integration test
 
   - job: macos
@@ -235,5 +235,5 @@ stages:
 
     - script: |
         cd test
-        python nni_test/nnitest/run_tests.py --config config/pr_tests.yml
+        python training_service/nnitest/run_tests.py --config training_service/config/pr_tests.yml
       displayName: Simple integration test

test/async_sharing_test/config.yml

-authorName: default
-experimentName: example_weight_sharing
-trialConcurrency: 3
-maxExecDuration: 1h
-maxTrialNum: 10
-#choice: local, remote, pai
-trainingServicePlatform: remote
-#choice: true, false
-useAnnotation: false
-multiThread: true
-tuner:
-  codeDir: .
-  classFileName: simple_tuner.py
-  className: SimpleTuner
-trial:
-  command: python3 main.py
-  codeDir: .
-  gpuNum: 0
-machineList:
-  - ip: 10.10.10.10
-    username: bob
-    passwd: bob123
-  - ip: 10.10.10.11
-    username: bob
-    passwd: bob123

test/async_sharing_test/main.py

-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT license.
-
-"""
-Test code for weight sharing
-need NFS setup and mounted as `/mnt/nfs/nni`
-"""
-
-import hashlib
-import os
-import random
-import time
-
-import nni
-
-
-def generate_rand_file(fl_name):
-    """
-    generate random file and write to `fl_name`
-    """
-    fl_size = random.randint(1024, 102400)
-    fl_dir = os.path.split(fl_name)[0]
-    if not os.path.exists(fl_dir):
-        os.makedirs(fl_dir)
-    with open(fl_name, 'wb') as fout:
-        fout.write(os.urandom(fl_size))
-
-
-def check_sum(fl_name, tid=None):
-    """
-    compute checksum for generated file of `fl_name`
-    """
-    hasher = hashlib.md5()
-    with open(fl_name, 'rb') as fin:
-        for chunk in iter(lambda: fin.read(4096), b""):
-            hasher.update(chunk)
-    ret = hasher.hexdigest()
-    if tid is not None:
-        ret = ret + str(tid)
-    return ret
-
-
-if __name__ == '__main__':
-    nfs_path = '/mnt/nfs/nni/test'
-    params = nni.get_next_parameter()
-    print(params)
-    if params['id'] == 0:
-        model_file = os.path.join(nfs_path, str(params['id']), 'model.dat')
-        generate_rand_file(model_file)
-        time.sleep(10)
-        nni.report_final_result({
-            'checksum': check_sum(model_file, tid=params['id']),
-            'path': model_file
-        })
-    else:
-        model_file = params['prev_path']
-        time.sleep(10)
-        nni.report_final_result({
-            'checksum': check_sum(model_file, tid=params['prev_id'])
-        })

test/async_sharing_test/simple_tuner.py

-# Copyright (c) Microsoft Corporation.
-# Licensed under the MIT license.
-
-"""
-SimpleTuner for Weight Sharing
-"""
-
-import logging
-
-from threading import Event, Lock
-from nni.tuner import Tuner
-
-_logger = logging.getLogger('WeightSharingTuner')
-
-
-class SimpleTuner(Tuner):
-    """
-    simple tuner, test for weight sharing
-    """
-
-    def __init__(self):
-        super(SimpleTuner, self).__init__()
-        self.trial_meta = {}
-        self.f_id = None  # father
-        self.sig_event = Event()
-        self.thread_lock = Lock()
-
-    def generate_parameters(self, parameter_id, **kwargs):
-        if self.f_id is None:
-            self.thread_lock.acquire()
-            self.f_id = parameter_id
-            self.trial_meta[parameter_id] = {
-                'prev_id': 0,
-                'id': parameter_id,
-                'checksum': None,
-                'path': '',
-            }
-            _logger.info('generate parameter for father trial %s', parameter_id)
-            self.thread_lock.release()
-            return {
-                'prev_id': 0,
-                'id': parameter_id,
-            }
-        else:
-            self.sig_event.wait()
-            self.thread_lock.acquire()
-            self.trial_meta[parameter_id] = {
-                'id': parameter_id,
-                'prev_id': self.f_id,
-                'prev_path': self.trial_meta[self.f_id]['path']
-            }
-            self.thread_lock.release()
-            return self.trial_meta[parameter_id]
-
-    def receive_trial_result(self, parameter_id, parameters, reward, **kwargs):
-        self.thread_lock.acquire()
-        if parameter_id == self.f_id:
-            self.trial_meta[parameter_id]['checksum'] = reward['checksum']
-            self.trial_meta[parameter_id]['path'] = reward['path']
-            self.sig_event.set()
-        else:
-            if reward['checksum'] != self.trial_meta[self.f_id]['checksum']:
-                raise ValueError("Inconsistency in weight sharing: {} != {}".format(
-                    reward['checksum'], self.trial_meta[self.f_id]['checksum']))
-        self.thread_lock.release()
-
-    def update_search_space(self, search_space):
-        pass

test/nni_test/setup.py

-from setuptools import setup, find_packages
-
-setup(
-    name="nnitest",
-    version="0.0.1",
-    author = 'Microsoft NNI team',
-    author_email = 'nni@microsoft.com',
-    description = 'Neural Network Intelligence package',
-    license = 'MIT',
-    url = 'https://github.com/Microsoft/nni',
-    packages=find_packages('nnitest'),
-    long_description="",
-    classifiers = [
-        'Programming Language :: Python :: 3',
-        'License :: OSI Approved :: MIT License',
-        "Operating System :: OS Independent"
-    ],
-)

test/retiarii_test/README.md

+Will be merged into algo/unittest in future.

test/scripts/README.md

+Will be moved to pipeline yamls in future.

test/config/customized_tuners/demotuner-sklearn-classification-v2.yml → test/training_service/config/customized_tuners/demotuner-sklearn-classification-v2.yml

 experimentName: default_test
-searchSpaceFile: ../../../examples/trials/sklearn/classification/search_space.json
+searchSpaceFile: ../../../../examples/trials/sklearn/classification/search_space.json
 trialCommand: python3 main.py
-trialCodeDirectory: ../../../examples/trials/sklearn/classification
+trialCodeDirectory: ../../../../examples/trials/sklearn/classification
 trialGpuNumber: 0
 trialConcurrency: 4
 maxExperimentDuration: 15m

test/config/customized_tuners/demotuner-sklearn-classification.yml → test/training_service/config/customized_tuners/demotuner-sklearn-classification.yml

@@ -3,7 +3,7 @@ experimentName: default_test
 maxExecDuration: 5m
 maxTrialNum: 4
 trialConcurrency: 2
-searchSpacePath: ../../../examples/trials/sklearn/classification/search_space.json
+searchSpacePath: ../../../../examples/trials/sklearn/classification/search_space.json
 
 tuner:
   builtinTunerName: demotuner
@@ -12,7 +12,7 @@ assessor:
   classArgs:
     optimize_mode: maximize
 trial:
-  codeDir: ../../../examples/trials/sklearn/classification
+  codeDir: ../../../../examples/trials/sklearn/classification
   command: python3 main.py
   gpuNum: 0
 

test/config/examples/cifar10-pytorch-v2.yml → test/training_service/config/examples/cifar10-pytorch-v2.yml

 experimentName: default_test
 searchSpaceFile: cifar10_search_space.json
 trialCommand: python3 main.py --epochs 1 --batches 1
-trialCodeDirectory: ../../../examples/trials/cifar10_pytorch
+trialCodeDirectory: ../../../../examples/trials/cifar10_pytorch
 trialGpuNumber: 0
 trialConcurrency: 1
 maxExperimentDuration: 15m

test/config/examples/cifar10-pytorch.yml → test/training_service/config/examples/cifar10-pytorch.yml

@@ -12,7 +12,7 @@ assessor:
   classArgs:
     optimize_mode: maximize
 trial:
-  codeDir: ../../../examples/trials/cifar10_pytorch
+  codeDir: ../../../../examples/trials/cifar10_pytorch
   command: python3 main.py --epochs 1 --batches 1
   gpuNum: 1
 

test/config/examples/classic-nas-pytorch-v2.yml → test/training_service/config/examples/classic-nas-pytorch-v2.yml

 experimentName: default_test
 searchSpaceFile: ni-nas-search-space.json
 trialCommand: python3 main.py --epochs 1 --batches 1
-trialCodeDirectory: ../../../examples/nas/legacy/classic_nas
+trialCodeDirectory: ../../../../examples/nas/legacy/classic_nas
 trialGpuNumber: 0
 trialConcurrency: 1
 maxExperimentDuration: 15m

test/config/examples/classic-nas-pytorch.yml → test/training_service/config/examples/classic-nas-pytorch.yml

@@ -11,7 +11,7 @@ tuner:
     optimize_mode: maximize
 trial:
   command: python3 mnist.py --epochs 1
-  codeDir: ../../../examples/nas/legacy/classic_nas
+  codeDir: ../../../../examples/nas/legacy/classic_nas
   gpuNum: 0
 
 useAnnotation: false