Skip to content

GitLab

Unverified Commit a8f86a78 authored by Yuge Zhang's avatar Yuge Zhang Committed by GitHub
Browse files

Refactor integration test (step 4) - refactor test file tree structure (#4895)

parent b0732e01
Show whitespace changes
Inline Side-by-side
Showing with 90 additions and 90 deletions
test/config/integration_tests.yml → test/training_service/config/integration_tests.yml
View file @ a8f86a78
...@@ -12,14 +12,14 @@ testCases: ...@@ -12,14 +12,14 @@ testCases:
####################################################################### #######################################################################
- name: sklearn-classification - name: sklearn-classification
# test case config yml file relative to nni source code directory # test case config yml file relative to nni source code directory
configFile: test/config/examples/sklearn-classification.yml configFile: test/training_service/config/examples/sklearn-classification.yml
# test case specific config, the content of configFile will be overrided # test case specific config, the content of configFile will be overrided
# by config section # by config section
config: config:
# validator is called after experiment is done # validator is called after experiment is done
# validator class needs to be implemented in nni_test/nnitest/validators.py # validator class needs to be implemented in training_service/nnitest/validators.py
validator: validator:
# launch command, default launch command is 'nnictl create --config $configFile' # launch command, default launch command is 'nnictl create --config $configFile'
...@@ -35,7 +35,7 @@ testCases: ...@@ -35,7 +35,7 @@ testCases:
experimentStatusCheck: True experimentStatusCheck: True
- name: shared-storage-remote-azureblob - name: shared-storage-remote-azureblob
configFile: test/config/sharedstorage_test/config_sharedstorage_remote_azureblob.yml configFile: test/training_service/config/sharedstorage_test/config_sharedstorage_remote_azureblob.yml
config: config:
sharedStorage: sharedStorage:
localMountPoint: /tmp/nnimount/testlocalrootpath localMountPoint: /tmp/nnimount/testlocalrootpath
...@@ -50,12 +50,12 @@ testCases: ...@@ -50,12 +50,12 @@ testCases:
rootpath: /tmp/nnimount/testlocalrootpath rootpath: /tmp/nnimount/testlocalrootpath
- name: sklearn-regression - name: sklearn-regression
configFile: test/config/examples/sklearn-regression.yml configFile: test/training_service/config/examples/sklearn-regression.yml
# mount point in local may late to umount when stop experiment in shared-storage-remote-azureblob, # mount point in local may late to umount when stop experiment in shared-storage-remote-azureblob,
# so keep two shared storage tests away from each other. # so keep two shared storage tests away from each other.
- name: shared-storage-remote-nfs - name: shared-storage-remote-nfs
configFile: test/config/sharedstorage_test/config_sharedstorage_remote_nfs.yml configFile: test/training_service/config/sharedstorage_test/config_sharedstorage_remote_nfs.yml
trainingService: remote trainingService: remote
validator: validator:
class: FileExistValidator class: FileExistValidator
...@@ -63,7 +63,7 @@ testCases: ...@@ -63,7 +63,7 @@ testCases:
rootpath: /tmp/nnimount/testlocalrootpath rootpath: /tmp/nnimount/testlocalrootpath
- name: mnist-tensorflow - name: mnist-tensorflow
configFile: test/config/examples/mnist-tfv2.yml configFile: test/training_service/config/examples/mnist-tfv2.yml
config: config:
maxExecDuration: 10m # This example will use longger time in remote mode, set max_duration to 10m to avoid timeout error. maxExecDuration: 10m # This example will use longger time in remote mode, set max_duration to 10m to avoid timeout error.
maxTrialNum: 1 maxTrialNum: 1
...@@ -71,32 +71,32 @@ testCases: ...@@ -71,32 +71,32 @@ testCases:
trainingService: local remote # FIXME: timeout on pai, looks like tensorflow failed to link CUDA trainingService: local remote # FIXME: timeout on pai, looks like tensorflow failed to link CUDA
- name: mnist-pytorch-local - name: mnist-pytorch-local
configFile: test/config/examples/mnist-pytorch.yml configFile: test/training_service/config/examples/mnist-pytorch.yml
# download data first, to prevent concurrent issue. # download data first, to prevent concurrent issue.
launchCommand: python3 ../examples/trials/mnist-pytorch/mnist.py --epochs 1 --batch_num 0 --data_dir ../examples/trials/mnist-pytorch/data && nnictl create --config $configFile --debug launchCommand: python3 ../examples/trials/mnist-pytorch/mnist.py --epochs 1 --batch_num 0 --data_dir ../examples/trials/mnist-pytorch/data && nnictl create --config $configFile --debug
trainingService: local trainingService: local
- name: mnist-pytorch-local-gpu - name: mnist-pytorch-local-gpu
configFile: test/config/examples/mnist-pytorch-gpu.yml configFile: test/training_service/config/examples/mnist-pytorch-gpu.yml
# download data first, to prevent concurrent issue. # download data first, to prevent concurrent issue.
launchCommand: python3 ../examples/trials/mnist-pytorch/mnist.py --epochs 1 --batch_num 0 --data_dir ../examples/trials/mnist-pytorch/data && nnictl create --config $configFile --debug launchCommand: python3 ../examples/trials/mnist-pytorch/mnist.py --epochs 1 --batch_num 0 --data_dir ../examples/trials/mnist-pytorch/data && nnictl create --config $configFile --debug
trainingService: local trainingService: local
- name: mnist-pytorch-non-local - name: mnist-pytorch-non-local
configFile: test/config/examples/mnist-pytorch.yml configFile: test/training_service/config/examples/mnist-pytorch.yml
# download data first, to prevent concurrent issue. # download data first, to prevent concurrent issue.
launchCommand: nnictl create --config $configFile --debug launchCommand: nnictl create --config $configFile --debug
trainingService: remote pai kubeflow frameworkcontroller dlts trainingService: remote pai kubeflow frameworkcontroller dlts
# TODO: move this and following commented test cases to pytorch or tf2 # TODO: move this and following commented test cases to pytorch or tf2
#- name: mnist-annotation #- name: mnist-annotation
# configFile: test/config/examples/mnist-annotation.yml # configFile: test/training_service/config/examples/mnist-annotation.yml
# config: # config:
# maxTrialNum: 1 # maxTrialNum: 1
# trialConcurrency: 1 # trialConcurrency: 1
- name: cifar10-pytorch - name: cifar10-pytorch
configFile: test/config/examples/cifar10-pytorch.yml configFile: test/training_service/config/examples/cifar10-pytorch.yml
config: config:
# this example downloads large pretrained model weights # this example downloads large pretrained model weights
# test 1 trial to save time # test 1 trial to save time
...@@ -108,31 +108,31 @@ testCases: ...@@ -108,31 +108,31 @@ testCases:
gpuNum: 0 gpuNum: 0
- name: cifar10-pytorch-adl - name: cifar10-pytorch-adl
configFile: test/config/examples/cifar10-pytorch-adl.yml configFile: test/training_service/config/examples/cifar10-pytorch-adl.yml
trainingService: adl trainingService: adl
#- name: nested-ss #- name: nested-ss
# configFile: test/config/examples/mnist-nested-search-space.yml # configFile: test/training_service/config/examples/mnist-nested-search-space.yml
- name: classic-nas-gen-ss - name: classic-nas-gen-ss
configFile: test/config/examples/classic-nas-pytorch.yml configFile: test/training_service/config/examples/classic-nas-pytorch.yml
launchCommand: nnictl ss_gen --trial_command="python3 mnist.py --epochs 1" --trial_dir=../examples/nas/legacy/classic_nas --file=config/examples/nni-nas-search-space.json launchCommand: nnictl ss_gen --trial_command="python3 mnist.py --epochs 1" --trial_dir=../examples/nas/legacy/classic_nas --file=training_service/config/examples/nni-nas-search-space.json
stopCommand: stopCommand:
experimentStatusCheck: False experimentStatusCheck: False
trainingService: local trainingService: local
- name: classic-nas-pytorch - name: classic-nas-pytorch
configFile: test/config/examples/classic-nas-pytorch.yml configFile: test/training_service/config/examples/classic-nas-pytorch.yml
# remove search space file # remove search space file
stopCommand: nnictl stop stopCommand: nnictl stop
onExitCommand: python3 -c "import os; os.remove('config/examples/nni-nas-search-space.json')" onExitCommand: python3 -c "import os; os.remove('training_service/config/examples/nni-nas-search-space.json')"
trainingService: local trainingService: local
######################################################################### #########################################################################
# nni features test # nni features test
######################################################################### #########################################################################
- name: metrics-float - name: metrics-float
configFile: test/config/metrics_test/config.yml configFile: test/training_service/config/metrics_test/config.yml
config: config:
maxTrialNum: 1 maxTrialNum: 1
trialConcurrency: 1 trialConcurrency: 1
...@@ -142,7 +142,7 @@ testCases: ...@@ -142,7 +142,7 @@ testCases:
expected_result_file: expected_metrics.json expected_result_file: expected_metrics.json
- name: export-float - name: export-float
configFile: test/config/metrics_test/config.yml configFile: test/training_service/config/metrics_test/config.yml
config: config:
maxTrialNum: 1 maxTrialNum: 1
trialConcurrency: 1 trialConcurrency: 1
...@@ -150,7 +150,7 @@ testCases: ...@@ -150,7 +150,7 @@ testCases:
class: ExportValidator class: ExportValidator
- name: metrics-dict - name: metrics-dict
configFile: test/config/metrics_test/config_dict_metrics.yml configFile: test/training_service/config/metrics_test/config_dict_metrics.yml
config: config:
maxTrialNum: 1 maxTrialNum: 1
trialConcurrency: 1 trialConcurrency: 1
...@@ -160,7 +160,7 @@ testCases: ...@@ -160,7 +160,7 @@ testCases:
expected_result_file: expected_metrics_dict.json expected_result_file: expected_metrics_dict.json
- name: export-dict - name: export-dict
configFile: test/config/metrics_test/config_dict_metrics.yml configFile: test/training_service/config/metrics_test/config_dict_metrics.yml
config: config:
maxTrialNum: 1 maxTrialNum: 1
trialConcurrency: 1 trialConcurrency: 1
...@@ -168,34 +168,34 @@ testCases: ...@@ -168,34 +168,34 @@ testCases:
class: ExportValidator class: ExportValidator
- name: experiment-import - name: experiment-import
configFile: test/config/nnictl_experiment/sklearn-classification.yml configFile: test/training_service/config/nnictl_experiment/sklearn-classification.yml
validator: validator:
class: ImportValidator class: ImportValidator
kwargs: kwargs:
import_data_file_path: config/nnictl_experiment/test_import.json import_data_file_path: training_service/config/nnictl_experiment/test_import.json
- name: foreground - name: foreground
configFile: test/config/examples/sklearn-regression.yml configFile: test/training_service/config/examples/sklearn-regression.yml
launchCommand: python3 nni_test/nnitest/foreground.py --config $configFile --timeout 45 launchCommand: python3 training_service/nnitest/foreground.py --config $configFile --timeout 45
stopCommand: stopCommand:
experimentStatusCheck: False experimentStatusCheck: False
platform: linux darwin platform: linux darwin
# Experiment resume test part 1 # Experiment resume test part 1
- name: nnictl-resume-1 - name: nnictl-resume-1
configFile: test/config/examples/sklearn-regression.yml configFile: test/training_service/config/examples/sklearn-regression.yml
setExperimentIdtoVar: $resumeExpId setExperimentIdtoVar: $resumeExpId
# for subfolder in codedir test # for subfolder in codedir test
launchCommand: python3 -c "import os; os.makedirs('../examples/trials/sklearn/regression/subfolder', exist_ok=True); open('../examples/trials/sklearn/regression/subfolder/subfile', 'a').close()" && nnictl create --config $configFile --debug launchCommand: python3 -c "import os; os.makedirs('../examples/trials/sklearn/regression/subfolder', exist_ok=True); open('../examples/trials/sklearn/regression/subfolder/subfile', 'a').close()" && nnictl create --config $configFile --debug
# Experiment resume test part 2 # Experiment resume test part 2
- name: nnictl-resume-2 - name: nnictl-resume-2
configFile: test/config/examples/sklearn-regression.yml configFile: test/training_service/config/examples/sklearn-regression.yml
launchCommand: nnictl resume $resumeExpId launchCommand: nnictl resume $resumeExpId
# Experiment view test # Experiment view test
- name: nnictl-view - name: nnictl-view
configFile: test/config/examples/sklearn-regression.yml configFile: test/training_service/config/examples/sklearn-regression.yml
launchCommand: nnictl view $resumeExpId launchCommand: nnictl view $resumeExpId
experimentStatusCheck: False experimentStatusCheck: False
...@@ -204,54 +204,54 @@ testCases: ...@@ -204,54 +204,54 @@ testCases:
# nni assessor test # nni assessor test
######################################################################### #########################################################################
- name: assessor-curvefitting - name: assessor-curvefitting
configFile: test/config/assessors/curvefitting.yml configFile: test/training_service/config/assessors/curvefitting.yml
- name: assessor-medianstop - name: assessor-medianstop
configFile: test/config/assessors/medianstop.yml configFile: test/training_service/config/assessors/medianstop.yml
######################################################################### #########################################################################
# nni tuners test # nni tuners test
######################################################################### #########################################################################
#- name: tuner-annel #- name: tuner-annel
# configFile: test/config/tuners/anneal.yml # configFile: test/training_service/config/tuners/anneal.yml
#- name: tuner-evolution #- name: tuner-evolution
# configFile: test/config/tuners/evolution.yml # configFile: test/training_service/config/tuners/evolution.yml
#- name: tuner-random #- name: tuner-random
# configFile: test/config/tuners/random.yml # configFile: test/training_service/config/tuners/random.yml
#- name: tuner-smac #- name: tuner-smac
# configFile: test/config/tuners/smac.yml # configFile: test/training_service/config/tuners/smac.yml
# platform: linux darwin # platform: linux darwin
#- name: tuner-tpe #- name: tuner-tpe
# configFile: test/config/tuners/tpe.yml # configFile: test/training_service/config/tuners/tpe.yml
#- name: tuner-batch #- name: tuner-batch
# configFile: test/config/tuners/batch.yml # configFile: test/training_service/config/tuners/batch.yml
#- name: tuner-bohb #- name: tuner-bohb
# configFile: test/config/tuners/bohb.yml # configFile: test/training_service/config/tuners/bohb.yml
# platform: linux darwin # platform: linux darwin
#- name: tuner-gp #- name: tuner-gp
# configFile: test/config/tuners/gp.yml # configFile: test/training_service/config/tuners/gp.yml
#- name: tuner-grid #- name: tuner-grid
# configFile: test/config/tuners/gridsearch.yml # configFile: test/training_service/config/tuners/gridsearch.yml
#- name: tuner-hyperband #- name: tuner-hyperband
# configFile: test/config/tuners/hyperband.yml # configFile: test/training_service/config/tuners/hyperband.yml
#- name: tuner-metis #- name: tuner-metis
# configFile: test/config/tuners/metis.yml # configFile: test/training_service/config/tuners/metis.yml
- name: tuner-regularized_evolution - name: tuner-regularized_evolution
configFile: test/config/tuners/regularized_evolution_tuner.yml configFile: test/training_service/config/tuners/regularized_evolution_tuner.yml
######################################################################### #########################################################################
# nni customized-tuners test # nni customized-tuners test
######################################################################### #########################################################################
- name: customized-tuners-demotuner - name: customized-tuners-demotuner
configFile: test/config/customized_tuners/demotuner-sklearn-classification.yml configFile: test/training_service/config/customized_tuners/demotuner-sklearn-classification.yml
test/config/integration_tests_config_v2.yml → test/training_service/config/integration_tests_config_v2.yml
View file @ a8f86a78
...@@ -12,102 +12,102 @@ testCases: ...@@ -12,102 +12,102 @@ testCases:
####################################################################### #######################################################################
- name: sklearn-classification - name: sklearn-classification
# test case config yml file relative to nni source code directory # test case config yml file relative to nni source code directory
configFile: test/config/examples/sklearn-classification-v2.yml configFile: test/training_service/config/examples/sklearn-classification-v2.yml
- name: sklearn-regression - name: sklearn-regression
configFile: test/config/examples/sklearn-regression-v2.yml configFile: test/training_service/config/examples/sklearn-regression-v2.yml
- name: mnist-tensorflow - name: mnist-tensorflow
configFile: test/config/examples/mnist-tfv2-v2.yml configFile: test/training_service/config/examples/mnist-tfv2-v2.yml
trainingService: local remote trainingService: local remote
- name: mnist-pytorch-local - name: mnist-pytorch-local
configFile: test/config/examples/mnist-pytorch-v2.yml configFile: test/training_service/config/examples/mnist-pytorch-v2.yml
# download data first, to prevent concurrent issue. # download data first, to prevent concurrent issue.
launchCommand: python3 ../examples/trials/mnist-pytorch/mnist.py --epochs 1 --batch_num 0 --data_dir ../examples/trials/mnist-pytorch/data && nnictl create --config $configFile --debug launchCommand: python3 ../examples/trials/mnist-pytorch/mnist.py --epochs 1 --batch_num 0 --data_dir ../examples/trials/mnist-pytorch/data && nnictl create --config $configFile --debug
trainingService: local trainingService: local
- name: mnist-pytorch-non-local - name: mnist-pytorch-non-local
configFile: test/config/examples/mnist-pytorch-v2.yml configFile: test/training_service/config/examples/mnist-pytorch-v2.yml
trainingService: remote pai kubeflow frameworkcontroller dlts hybrid trainingService: remote pai kubeflow frameworkcontroller dlts hybrid
- name: cifar10-pytorch - name: cifar10-pytorch
configFile: test/config/examples/cifar10-pytorch-v2.yml configFile: test/training_service/config/examples/cifar10-pytorch-v2.yml
- name: cifar10-pytorch-adl - name: cifar10-pytorch-adl
configFile: test/config/examples/cifar10-pytorch-adl.yml configFile: test/training_service/config/examples/cifar10-pytorch-adl.yml
trainingService: adl trainingService: adl
- name: classic-nas-gen-ss - name: classic-nas-gen-ss
configFile: test/config/examples/classic-nas-pytorch-v2.yml configFile: test/training_service/config/examples/classic-nas-pytorch-v2.yml
launchCommand: nnictl ss_gen --trial_command="python3 mnist.py --epochs 1" --trial_dir=../examples/nas/legacy/classic_nas --file=config/examples/nni-nas-search-space.json launchCommand: nnictl ss_gen --trial_command="python3 mnist.py --epochs 1" --trial_dir=../examples/nas/legacy/classic_nas --file=training_service/config/examples/nni-nas-search-space.json
stopCommand: stopCommand:
experimentStatusCheck: False experimentStatusCheck: False
trainingService: local trainingService: local
- name: classic-nas-pytorch - name: classic-nas-pytorch
configFile: test/config/examples/classic-nas-pytorch-v2.yml configFile: test/training_service/config/examples/classic-nas-pytorch-v2.yml
# remove search space file # remove search space file
stopCommand: nnictl stop stopCommand: nnictl stop
onExitCommand: python3 -c "import os; os.remove('config/examples/nni-nas-search-space.json')" onExitCommand: python3 -c "import os; os.remove('training_service/examples/nni-nas-search-space.json')"
trainingService: local trainingService: local
######################################################################### #########################################################################
# nni features test # nni features test
######################################################################### #########################################################################
- name: metrics-float - name: metrics-float
configFile: test/config/metrics_test/config-v2.yml configFile: test/training_service/config/metrics_test/config-v2.yml
validator: validator:
class: MetricsValidator class: MetricsValidator
kwargs: kwargs:
expected_result_file: expected_metrics.json expected_result_file: expected_metrics.json
- name: export-float - name: export-float
configFile: test/config/metrics_test/config-v2.yml configFile: test/training_service/config/metrics_test/config-v2.yml
validator: validator:
class: ExportValidator class: ExportValidator
- name: metrics-dict - name: metrics-dict
configFile: test/config/metrics_test/config_dict_metrics-v2.yml configFile: test/training_service/config/metrics_test/config_dict_metrics-v2.yml
validator: validator:
class: MetricsValidator class: MetricsValidator
kwargs: kwargs:
expected_result_file: expected_metrics_dict.json expected_result_file: expected_metrics_dict.json
- name: export-dict - name: export-dict
configFile: test/config/metrics_test/config_dict_metrics-v2.yml configFile: test/training_service/config/metrics_test/config_dict_metrics-v2.yml
validator: validator:
class: ExportValidator class: ExportValidator
- name: experiment-import - name: experiment-import
configFile: test/config/nnictl_experiment/sklearn-classification-v2.yml configFile: test/training_service/config/nnictl_experiment/sklearn-classification-v2.yml
validator: validator:
class: ImportValidator class: ImportValidator
kwargs: kwargs:
import_data_file_path: config/nnictl_experiment/test_import.json import_data_file_path: training_service/config/nnictl_experiment/test_import.json
- name: foreground - name: foreground
configFile: test/config/examples/sklearn-regression-v2.yml configFile: test/training_service/config/examples/sklearn-regression-v2.yml
launchCommand: python3 nni_test/nnitest/foreground.py --config $configFile --timeout 45 launchCommand: python3 training_service/nnitest/foreground.py --config $configFile --timeout 45
stopCommand: stopCommand:
experimentStatusCheck: False experimentStatusCheck: False
platform: linux darwin platform: linux darwin
# Experiment resume test part 1 # Experiment resume test part 1
- name: nnictl-resume-1 - name: nnictl-resume-1
configFile: test/config/examples/sklearn-regression-v2.yml configFile: test/training_service/config/examples/sklearn-regression-v2.yml
setExperimentIdtoVar: $resumeExpId setExperimentIdtoVar: $resumeExpId
# for subfolder in codedir test # for subfolder in codedir test
launchCommand: python3 -c "import os; os.makedirs('../examples/trials/sklearn/regression/subfolder', exist_ok=True); open('../examples/trials/sklearn/regression/subfolder/subfile', 'a').close()" && nnictl create --config $configFile --debug launchCommand: python3 -c "import os; os.makedirs('../examples/trials/sklearn/regression/subfolder', exist_ok=True); open('../examples/trials/sklearn/regression/subfolder/subfile', 'a').close()" && nnictl create --config $configFile --debug
# Experiment resume test part 2 # Experiment resume test part 2
- name: nnictl-resume-2 - name: nnictl-resume-2
configFile: test/config/examples/sklearn-regression-v2.yml configFile: test/training_service/config/examples/sklearn-regression-v2.yml
launchCommand: nnictl resume $resumeExpId launchCommand: nnictl resume $resumeExpId
# Experiment view test # Experiment view test
- name: nnictl-view - name: nnictl-view
configFile: test/config/examples/sklearn-regression-v2.yml configFile: test/training_service/config/examples/sklearn-regression-v2.yml
launchCommand: nnictl view $resumeExpId launchCommand: nnictl view $resumeExpId
experimentStatusCheck: False experimentStatusCheck: False
...@@ -116,20 +116,20 @@ testCases: ...@@ -116,20 +116,20 @@ testCases:
# nni assessor test # nni assessor test
######################################################################### #########################################################################
- name: assessor-curvefitting - name: assessor-curvefitting
configFile: test/config/assessors/curvefitting-v2.yml configFile: test/training_service/config/assessors/curvefitting-v2.yml
- name: assessor-medianstop - name: assessor-medianstop
configFile: test/config/assessors/medianstop-v2.yml configFile: test/training_service/config/assessors/medianstop-v2.yml
######################################################################### #########################################################################
# nni tuners test # nni tuners test
######################################################################### #########################################################################
- name: tuner-regularized_evolution - name: tuner-regularized_evolution
configFile: test/config/tuners/regularized_evolution_tuner-v2.yml configFile: test/training_service/config/tuners/regularized_evolution_tuner-v2.yml
######################################################################### #########################################################################
# nni customized-tuners test # nni customized-tuners test
######################################################################### #########################################################################
- name: customized-tuners-demotuner - name: customized-tuners-demotuner
configFile: test/config/customized_tuners/demotuner-sklearn-classification-v2.yml configFile: test/training_service/config/customized_tuners/demotuner-sklearn-classification-v2.yml
test/config/integration_tests_tf2.yml → test/training_service/config/integration_tests_tf2.yml
View file @ a8f86a78
...@@ -12,14 +12,14 @@ testCases: ...@@ -12,14 +12,14 @@ testCases:
####################################################################### #######################################################################
- name: sklearn-classification - name: sklearn-classification
# test case config yml file relative to nni source code directory # test case config yml file relative to nni source code directory
configFile: test/config/examples/sklearn-classification.yml configFile: test/training_service/config/examples/sklearn-classification.yml
# test case specific config, the content of configFile will be overrided # test case specific config, the content of configFile will be overrided
# by config section # by config section
config: config:
# validator is called after experiment is done # validator is called after experiment is done
# validator class needs to be implemented in nni_test/nnitest/validators.py # validator class needs to be implemented in training_service/nnitest/validators.py
validator: validator:
# launch command, default launch command is 'nnictl create --config $configFile' # launch command, default launch command is 'nnictl create --config $configFile'
...@@ -35,13 +35,13 @@ testCases: ...@@ -35,13 +35,13 @@ testCases:
experimentStatusCheck: True experimentStatusCheck: True
- name: sklearn-regression - name: sklearn-regression
configFile: test/config/examples/sklearn-regression.yml configFile: test/training_service/config/examples/sklearn-regression.yml
- name: mnist-pytorch - name: mnist-pytorch
configFile: test/config/examples/mnist-pytorch.yml configFile: test/training_service/config/examples/mnist-pytorch.yml
- name: cifar10-pytorch - name: cifar10-pytorch
configFile: test/config/examples/cifar10-pytorch.yml configFile: test/training_service/config/examples/cifar10-pytorch.yml
config: config:
# this example downloads large pretrained model weights # this example downloads large pretrained model weights
# test 1 trial to save time # test 1 trial to save time
...@@ -53,28 +53,28 @@ testCases: ...@@ -53,28 +53,28 @@ testCases:
gpuNum: 0 gpuNum: 0
- name: cifar10-pytorch-adl - name: cifar10-pytorch-adl
configFile: test/config/examples/cifar10-pytorch-adl.yml configFile: test/training_service/config/examples/cifar10-pytorch-adl.yml
trainingService: adl trainingService: adl
- name: classic-nas-gen-ss - name: classic-nas-gen-ss
configFile: test/config/examples/classic-nas-tf2.yml configFile: test/training_service/config/examples/classic-nas-tf2.yml
launchCommand: nnictl ss_gen --trial_command="python3 train.py --epochs 1" --trial_dir=../examples/nas/legacy/classic_nas-tf --file=config/examples/nni-nas-search-space-tf2.json launchCommand: nnictl ss_gen --trial_command="python3 train.py --epochs 1" --trial_dir=../examples/nas/legacy/classic_nas-tf --file=training_service/config/examples/nni-nas-search-space-tf2.json
stopCommand: stopCommand:
experimentStatusCheck: False experimentStatusCheck: False
trainingService: local trainingService: local
- name: classic-nas-tensorflow2 - name: classic-nas-tensorflow2
configFile: test/config/examples/classic-nas-tf2.yml configFile: test/training_service/config/examples/classic-nas-tf2.yml
# remove search space file # remove search space file
stopCommand: nnictl stop stopCommand: nnictl stop
onExitCommand: python3 -c 'import os; os.remove("config/examples/nni-nas-search-space-tf2.json")' onExitCommand: python3 -c 'import os; os.remove("training_service/config/examples/nni-nas-search-space-tf2.json")'
trainingService: local trainingService: local
######################################################################### #########################################################################
# nni features test # nni features test
######################################################################### #########################################################################
- name: metrics-float - name: metrics-float
configFile: test/config/metrics_test/config.yml configFile: test/training_service/config/metrics_test/config.yml
config: config:
maxTrialNum: 1 maxTrialNum: 1
trialConcurrency: 1 trialConcurrency: 1
...@@ -84,7 +84,7 @@ testCases: ...@@ -84,7 +84,7 @@ testCases:
expected_result_file: expected_metrics.json expected_result_file: expected_metrics.json
- name: export-float - name: export-float
configFile: test/config/metrics_test/config.yml configFile: test/training_service/config/metrics_test/config.yml
config: config:
maxTrialNum: 1 maxTrialNum: 1
trialConcurrency: 1 trialConcurrency: 1
...@@ -92,7 +92,7 @@ testCases: ...@@ -92,7 +92,7 @@ testCases:
class: ExportValidator class: ExportValidator
- name: metrics-dict - name: metrics-dict
configFile: test/config/metrics_test/config_dict_metrics.yml configFile: test/training_service/config/metrics_test/config_dict_metrics.yml
config: config:
maxTrialNum: 1 maxTrialNum: 1
trialConcurrency: 1 trialConcurrency: 1
...@@ -102,7 +102,7 @@ testCases: ...@@ -102,7 +102,7 @@ testCases:
expected_result_file: expected_metrics_dict.json expected_result_file: expected_metrics_dict.json
- name: export-dict - name: export-dict
configFile: test/config/metrics_test/config_dict_metrics.yml configFile: test/training_service/config/metrics_test/config_dict_metrics.yml
config: config:
maxTrialNum: 1 maxTrialNum: 1
trialConcurrency: 1 trialConcurrency: 1
...@@ -110,25 +110,25 @@ testCases: ...@@ -110,25 +110,25 @@ testCases:
class: ExportValidator class: ExportValidator
- name: foreground - name: foreground
configFile: test/config/examples/sklearn-regression.yml configFile: test/training_service/config/examples/sklearn-regression.yml
launchCommand: python3 nni_test/nnitest/foreground.py --config $configFile --timeout 45 launchCommand: python3 training_service/nnitest/foreground.py --config $configFile --timeout 45
stopCommand: stopCommand:
experimentStatusCheck: False experimentStatusCheck: False
platform: linux darwin platform: linux darwin
# Experiment resume test part 1 # Experiment resume test part 1
- name: nnictl-resume-1 - name: nnictl-resume-1
configFile: test/config/examples/sklearn-regression.yml configFile: test/training_service/config/examples/sklearn-regression.yml
setExperimentIdtoVar: $resumeExpId setExperimentIdtoVar: $resumeExpId
# Experiment resume test part 2 # Experiment resume test part 2
- name: nnictl-resume-2 - name: nnictl-resume-2
configFile: test/config/examples/sklearn-regression.yml configFile: test/training_service/config/examples/sklearn-regression.yml
launchCommand: nnictl resume $resumeExpId launchCommand: nnictl resume $resumeExpId
# Experiment view test # Experiment view test
- name: nnictl-view - name: nnictl-view
configFile: test/config/examples/sklearn-regression.yml configFile: test/training_service/config/examples/sklearn-regression.yml
launchCommand: nnictl view $resumeExpId launchCommand: nnictl view $resumeExpId
experimentStatusCheck: False experimentStatusCheck: False
...@@ -136,7 +136,7 @@ testCases: ...@@ -136,7 +136,7 @@ testCases:
# nni assessor test # nni assessor test
######################################################################### #########################################################################
- name: assessor-curvefitting - name: assessor-curvefitting
configFile: test/config/assessors/curvefitting.yml configFile: test/training_service/config/assessors/curvefitting.yml
- name: assessor-medianstop - name: assessor-medianstop
configFile: test/config/assessors/medianstop.yml configFile: test/training_service/config/assessors/medianstop.yml
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment