Skip to content

GitLab

Unverified Commit a8f86a78 authored by Yuge Zhang's avatar Yuge Zhang Committed by GitHub
Browse files

Refactor integration test (step 4) - refactor test file tree structure (#4895)

parent b0732e01
Hide whitespace changes
Inline Side-by-side
Showing with 90 additions and 90 deletions
test/config/integration_tests.yml → test/training_service/config/integration_tests.yml
View file @ a8f86a78
......@@ -12,14 +12,14 @@ testCases:
#######################################################################
- name: sklearn-classification
# test case config yml file relative to nni source code directory
configFile: test/config/examples/sklearn-classification.yml
configFile: test/training_service/config/examples/sklearn-classification.yml
# test case specific config, the content of configFile will be overrided
# by config section
config:
# validator is called after experiment is done
# validator class needs to be implemented in nni_test/nnitest/validators.py
# validator class needs to be implemented in training_service/nnitest/validators.py
validator:
# launch command, default launch command is 'nnictl create --config $configFile'
......@@ -35,7 +35,7 @@ testCases:
experimentStatusCheck: True
- name: shared-storage-remote-azureblob
configFile: test/config/sharedstorage_test/config_sharedstorage_remote_azureblob.yml
configFile: test/training_service/config/sharedstorage_test/config_sharedstorage_remote_azureblob.yml
config:
sharedStorage:
localMountPoint: /tmp/nnimount/testlocalrootpath
......@@ -50,12 +50,12 @@ testCases:
rootpath: /tmp/nnimount/testlocalrootpath
- name: sklearn-regression
configFile: test/config/examples/sklearn-regression.yml
configFile: test/training_service/config/examples/sklearn-regression.yml
# mount point in local may late to umount when stop experiment in shared-storage-remote-azureblob,
# so keep two shared storage tests away from each other.
- name: shared-storage-remote-nfs
configFile: test/config/sharedstorage_test/config_sharedstorage_remote_nfs.yml
configFile: test/training_service/config/sharedstorage_test/config_sharedstorage_remote_nfs.yml
trainingService: remote
validator:
class: FileExistValidator
......@@ -63,7 +63,7 @@ testCases:
rootpath: /tmp/nnimount/testlocalrootpath
- name: mnist-tensorflow
configFile: test/config/examples/mnist-tfv2.yml
configFile: test/training_service/config/examples/mnist-tfv2.yml
config:
maxExecDuration: 10m # This example will use longger time in remote mode, set max_duration to 10m to avoid timeout error.
maxTrialNum: 1
......@@ -71,32 +71,32 @@ testCases:
trainingService: local remote # FIXME: timeout on pai, looks like tensorflow failed to link CUDA
- name: mnist-pytorch-local
configFile: test/config/examples/mnist-pytorch.yml
configFile: test/training_service/config/examples/mnist-pytorch.yml
# download data first, to prevent concurrent issue.
launchCommand: python3 ../examples/trials/mnist-pytorch/mnist.py --epochs 1 --batch_num 0 --data_dir ../examples/trials/mnist-pytorch/data && nnictl create --config $configFile --debug
trainingService: local
- name: mnist-pytorch-local-gpu
configFile: test/config/examples/mnist-pytorch-gpu.yml
configFile: test/training_service/config/examples/mnist-pytorch-gpu.yml
# download data first, to prevent concurrent issue.
launchCommand: python3 ../examples/trials/mnist-pytorch/mnist.py --epochs 1 --batch_num 0 --data_dir ../examples/trials/mnist-pytorch/data && nnictl create --config $configFile --debug
trainingService: local
- name: mnist-pytorch-non-local
configFile: test/config/examples/mnist-pytorch.yml
configFile: test/training_service/config/examples/mnist-pytorch.yml
# download data first, to prevent concurrent issue.
launchCommand: nnictl create --config $configFile --debug
trainingService: remote pai kubeflow frameworkcontroller dlts
# TODO: move this and following commented test cases to pytorch or tf2
#- name: mnist-annotation
# configFile: test/config/examples/mnist-annotation.yml
# configFile: test/training_service/config/examples/mnist-annotation.yml
# config:
# maxTrialNum: 1
# trialConcurrency: 1
- name: cifar10-pytorch
configFile: test/config/examples/cifar10-pytorch.yml
configFile: test/training_service/config/examples/cifar10-pytorch.yml
config:
# this example downloads large pretrained model weights
# test 1 trial to save time
......@@ -108,31 +108,31 @@ testCases:
gpuNum: 0
- name: cifar10-pytorch-adl
configFile: test/config/examples/cifar10-pytorch-adl.yml
configFile: test/training_service/config/examples/cifar10-pytorch-adl.yml
trainingService: adl
#- name: nested-ss
# configFile: test/config/examples/mnist-nested-search-space.yml
# configFile: test/training_service/config/examples/mnist-nested-search-space.yml
- name: classic-nas-gen-ss
configFile: test/config/examples/classic-nas-pytorch.yml
launchCommand: nnictl ss_gen --trial_command="python3 mnist.py --epochs 1" --trial_dir=../examples/nas/legacy/classic_nas --file=config/examples/nni-nas-search-space.json
configFile: test/training_service/config/examples/classic-nas-pytorch.yml
launchCommand: nnictl ss_gen --trial_command="python3 mnist.py --epochs 1" --trial_dir=../examples/nas/legacy/classic_nas --file=training_service/config/examples/nni-nas-search-space.json
stopCommand:
experimentStatusCheck: False
trainingService: local
- name: classic-nas-pytorch
configFile: test/config/examples/classic-nas-pytorch.yml
configFile: test/training_service/config/examples/classic-nas-pytorch.yml
# remove search space file
stopCommand: nnictl stop
onExitCommand: python3 -c "import os; os.remove('config/examples/nni-nas-search-space.json')"
onExitCommand: python3 -c "import os; os.remove('training_service/config/examples/nni-nas-search-space.json')"
trainingService: local
#########################################################################
# nni features test
#########################################################################
- name: metrics-float
configFile: test/config/metrics_test/config.yml
configFile: test/training_service/config/metrics_test/config.yml
config:
maxTrialNum: 1
trialConcurrency: 1
......@@ -142,7 +142,7 @@ testCases:
expected_result_file: expected_metrics.json
- name: export-float
configFile: test/config/metrics_test/config.yml
configFile: test/training_service/config/metrics_test/config.yml
config:
maxTrialNum: 1
trialConcurrency: 1
......@@ -150,7 +150,7 @@ testCases:
class: ExportValidator
- name: metrics-dict
configFile: test/config/metrics_test/config_dict_metrics.yml
configFile: test/training_service/config/metrics_test/config_dict_metrics.yml
config:
maxTrialNum: 1
trialConcurrency: 1
......@@ -160,7 +160,7 @@ testCases:
expected_result_file: expected_metrics_dict.json
- name: export-dict
configFile: test/config/metrics_test/config_dict_metrics.yml
configFile: test/training_service/config/metrics_test/config_dict_metrics.yml
config:
maxTrialNum: 1
trialConcurrency: 1
......@@ -168,34 +168,34 @@ testCases:
class: ExportValidator
- name: experiment-import
configFile: test/config/nnictl_experiment/sklearn-classification.yml
configFile: test/training_service/config/nnictl_experiment/sklearn-classification.yml
validator:
class: ImportValidator
kwargs:
import_data_file_path: config/nnictl_experiment/test_import.json
import_data_file_path: training_service/config/nnictl_experiment/test_import.json
- name: foreground
configFile: test/config/examples/sklearn-regression.yml
launchCommand: python3 nni_test/nnitest/foreground.py --config $configFile --timeout 45
configFile: test/training_service/config/examples/sklearn-regression.yml
launchCommand: python3 training_service/nnitest/foreground.py --config $configFile --timeout 45
stopCommand:
experimentStatusCheck: False
platform: linux darwin
# Experiment resume test part 1
- name: nnictl-resume-1
configFile: test/config/examples/sklearn-regression.yml
configFile: test/training_service/config/examples/sklearn-regression.yml
setExperimentIdtoVar: $resumeExpId
# for subfolder in codedir test
launchCommand: python3 -c "import os; os.makedirs('../examples/trials/sklearn/regression/subfolder', exist_ok=True); open('../examples/trials/sklearn/regression/subfolder/subfile', 'a').close()" && nnictl create --config $configFile --debug
# Experiment resume test part 2
- name: nnictl-resume-2
configFile: test/config/examples/sklearn-regression.yml
configFile: test/training_service/config/examples/sklearn-regression.yml
launchCommand: nnictl resume $resumeExpId
# Experiment view test
- name: nnictl-view
configFile: test/config/examples/sklearn-regression.yml
configFile: test/training_service/config/examples/sklearn-regression.yml
launchCommand: nnictl view $resumeExpId
experimentStatusCheck: False
......@@ -204,54 +204,54 @@ testCases:
# nni assessor test
#########################################################################
- name: assessor-curvefitting
configFile: test/config/assessors/curvefitting.yml
configFile: test/training_service/config/assessors/curvefitting.yml
- name: assessor-medianstop
configFile: test/config/assessors/medianstop.yml
configFile: test/training_service/config/assessors/medianstop.yml
#########################################################################
# nni tuners test
#########################################################################
#- name: tuner-annel
# configFile: test/config/tuners/anneal.yml
# configFile: test/training_service/config/tuners/anneal.yml
#- name: tuner-evolution
# configFile: test/config/tuners/evolution.yml
# configFile: test/training_service/config/tuners/evolution.yml
#- name: tuner-random
# configFile: test/config/tuners/random.yml
# configFile: test/training_service/config/tuners/random.yml
#- name: tuner-smac
# configFile: test/config/tuners/smac.yml
# configFile: test/training_service/config/tuners/smac.yml
# platform: linux darwin
#- name: tuner-tpe
# configFile: test/config/tuners/tpe.yml
# configFile: test/training_service/config/tuners/tpe.yml
#- name: tuner-batch
# configFile: test/config/tuners/batch.yml
# configFile: test/training_service/config/tuners/batch.yml
#- name: tuner-bohb
# configFile: test/config/tuners/bohb.yml
# configFile: test/training_service/config/tuners/bohb.yml
# platform: linux darwin
#- name: tuner-gp
# configFile: test/config/tuners/gp.yml
# configFile: test/training_service/config/tuners/gp.yml
#- name: tuner-grid
# configFile: test/config/tuners/gridsearch.yml
# configFile: test/training_service/config/tuners/gridsearch.yml
#- name: tuner-hyperband
# configFile: test/config/tuners/hyperband.yml
# configFile: test/training_service/config/tuners/hyperband.yml
#- name: tuner-metis
# configFile: test/config/tuners/metis.yml
# configFile: test/training_service/config/tuners/metis.yml
- name: tuner-regularized_evolution
configFile: test/config/tuners/regularized_evolution_tuner.yml
configFile: test/training_service/config/tuners/regularized_evolution_tuner.yml
#########################################################################
# nni customized-tuners test
#########################################################################
- name: customized-tuners-demotuner
configFile: test/config/customized_tuners/demotuner-sklearn-classification.yml
configFile: test/training_service/config/customized_tuners/demotuner-sklearn-classification.yml
test/config/integration_tests_config_v2.yml → test/training_service/config/integration_tests_config_v2.yml
View file @ a8f86a78
......@@ -12,102 +12,102 @@ testCases:
#######################################################################
- name: sklearn-classification
# test case config yml file relative to nni source code directory
configFile: test/config/examples/sklearn-classification-v2.yml
configFile: test/training_service/config/examples/sklearn-classification-v2.yml
- name: sklearn-regression
configFile: test/config/examples/sklearn-regression-v2.yml
configFile: test/training_service/config/examples/sklearn-regression-v2.yml
- name: mnist-tensorflow
configFile: test/config/examples/mnist-tfv2-v2.yml
configFile: test/training_service/config/examples/mnist-tfv2-v2.yml
trainingService: local remote
- name: mnist-pytorch-local
configFile: test/config/examples/mnist-pytorch-v2.yml
configFile: test/training_service/config/examples/mnist-pytorch-v2.yml
# download data first, to prevent concurrent issue.
launchCommand: python3 ../examples/trials/mnist-pytorch/mnist.py --epochs 1 --batch_num 0 --data_dir ../examples/trials/mnist-pytorch/data && nnictl create --config $configFile --debug
trainingService: local
- name: mnist-pytorch-non-local
configFile: test/config/examples/mnist-pytorch-v2.yml
configFile: test/training_service/config/examples/mnist-pytorch-v2.yml
trainingService: remote pai kubeflow frameworkcontroller dlts hybrid
- name: cifar10-pytorch
configFile: test/config/examples/cifar10-pytorch-v2.yml
configFile: test/training_service/config/examples/cifar10-pytorch-v2.yml
- name: cifar10-pytorch-adl
configFile: test/config/examples/cifar10-pytorch-adl.yml
configFile: test/training_service/config/examples/cifar10-pytorch-adl.yml
trainingService: adl
- name: classic-nas-gen-ss
configFile: test/config/examples/classic-nas-pytorch-v2.yml
launchCommand: nnictl ss_gen --trial_command="python3 mnist.py --epochs 1" --trial_dir=../examples/nas/legacy/classic_nas --file=config/examples/nni-nas-search-space.json
configFile: test/training_service/config/examples/classic-nas-pytorch-v2.yml
launchCommand: nnictl ss_gen --trial_command="python3 mnist.py --epochs 1" --trial_dir=../examples/nas/legacy/classic_nas --file=training_service/config/examples/nni-nas-search-space.json
stopCommand:
experimentStatusCheck: False
trainingService: local
- name: classic-nas-pytorch
configFile: test/config/examples/classic-nas-pytorch-v2.yml
configFile: test/training_service/config/examples/classic-nas-pytorch-v2.yml
# remove search space file
stopCommand: nnictl stop
onExitCommand: python3 -c "import os; os.remove('config/examples/nni-nas-search-space.json')"
onExitCommand: python3 -c "import os; os.remove('training_service/examples/nni-nas-search-space.json')"
trainingService: local
#########################################################################
# nni features test
#########################################################################
- name: metrics-float
configFile: test/config/metrics_test/config-v2.yml
configFile: test/training_service/config/metrics_test/config-v2.yml
validator:
class: MetricsValidator
kwargs:
expected_result_file: expected_metrics.json
- name: export-float
configFile: test/config/metrics_test/config-v2.yml
configFile: test/training_service/config/metrics_test/config-v2.yml
validator:
class: ExportValidator
- name: metrics-dict
configFile: test/config/metrics_test/config_dict_metrics-v2.yml
configFile: test/training_service/config/metrics_test/config_dict_metrics-v2.yml
validator:
class: MetricsValidator
kwargs:
expected_result_file: expected_metrics_dict.json
- name: export-dict
configFile: test/config/metrics_test/config_dict_metrics-v2.yml
configFile: test/training_service/config/metrics_test/config_dict_metrics-v2.yml
validator:
class: ExportValidator
- name: experiment-import
configFile: test/config/nnictl_experiment/sklearn-classification-v2.yml
configFile: test/training_service/config/nnictl_experiment/sklearn-classification-v2.yml
validator:
class: ImportValidator
kwargs:
import_data_file_path: config/nnictl_experiment/test_import.json
import_data_file_path: training_service/config/nnictl_experiment/test_import.json
- name: foreground
configFile: test/config/examples/sklearn-regression-v2.yml
launchCommand: python3 nni_test/nnitest/foreground.py --config $configFile --timeout 45
configFile: test/training_service/config/examples/sklearn-regression-v2.yml
launchCommand: python3 training_service/nnitest/foreground.py --config $configFile --timeout 45
stopCommand:
experimentStatusCheck: False
platform: linux darwin
# Experiment resume test part 1
- name: nnictl-resume-1
configFile: test/config/examples/sklearn-regression-v2.yml
configFile: test/training_service/config/examples/sklearn-regression-v2.yml
setExperimentIdtoVar: $resumeExpId
# for subfolder in codedir test
launchCommand: python3 -c "import os; os.makedirs('../examples/trials/sklearn/regression/subfolder', exist_ok=True); open('../examples/trials/sklearn/regression/subfolder/subfile', 'a').close()" && nnictl create --config $configFile --debug
# Experiment resume test part 2
- name: nnictl-resume-2
configFile: test/config/examples/sklearn-regression-v2.yml
configFile: test/training_service/config/examples/sklearn-regression-v2.yml
launchCommand: nnictl resume $resumeExpId
# Experiment view test
- name: nnictl-view
configFile: test/config/examples/sklearn-regression-v2.yml
configFile: test/training_service/config/examples/sklearn-regression-v2.yml
launchCommand: nnictl view $resumeExpId
experimentStatusCheck: False
......@@ -116,20 +116,20 @@ testCases:
# nni assessor test
#########################################################################
- name: assessor-curvefitting
configFile: test/config/assessors/curvefitting-v2.yml
configFile: test/training_service/config/assessors/curvefitting-v2.yml
- name: assessor-medianstop
configFile: test/config/assessors/medianstop-v2.yml
configFile: test/training_service/config/assessors/medianstop-v2.yml
#########################################################################
# nni tuners test
#########################################################################
- name: tuner-regularized_evolution
configFile: test/config/tuners/regularized_evolution_tuner-v2.yml
configFile: test/training_service/config/tuners/regularized_evolution_tuner-v2.yml
#########################################################################
# nni customized-tuners test
#########################################################################
- name: customized-tuners-demotuner
configFile: test/config/customized_tuners/demotuner-sklearn-classification-v2.yml
configFile: test/training_service/config/customized_tuners/demotuner-sklearn-classification-v2.yml
test/config/integration_tests_tf2.yml → test/training_service/config/integration_tests_tf2.yml
View file @ a8f86a78
......@@ -12,14 +12,14 @@ testCases:
#######################################################################
- name: sklearn-classification
# test case config yml file relative to nni source code directory
configFile: test/config/examples/sklearn-classification.yml
configFile: test/training_service/config/examples/sklearn-classification.yml
# test case specific config, the content of configFile will be overrided
# by config section
config:
# validator is called after experiment is done
# validator class needs to be implemented in nni_test/nnitest/validators.py
# validator class needs to be implemented in training_service/nnitest/validators.py
validator:
# launch command, default launch command is 'nnictl create --config $configFile'
......@@ -35,13 +35,13 @@ testCases:
experimentStatusCheck: True
- name: sklearn-regression
configFile: test/config/examples/sklearn-regression.yml
configFile: test/training_service/config/examples/sklearn-regression.yml
- name: mnist-pytorch
configFile: test/config/examples/mnist-pytorch.yml
configFile: test/training_service/config/examples/mnist-pytorch.yml
- name: cifar10-pytorch
configFile: test/config/examples/cifar10-pytorch.yml
configFile: test/training_service/config/examples/cifar10-pytorch.yml
config:
# this example downloads large pretrained model weights
# test 1 trial to save time
......@@ -53,28 +53,28 @@ testCases:
gpuNum: 0
- name: cifar10-pytorch-adl
configFile: test/config/examples/cifar10-pytorch-adl.yml
configFile: test/training_service/config/examples/cifar10-pytorch-adl.yml
trainingService: adl
- name: classic-nas-gen-ss
configFile: test/config/examples/classic-nas-tf2.yml
launchCommand: nnictl ss_gen --trial_command="python3 train.py --epochs 1" --trial_dir=../examples/nas/legacy/classic_nas-tf --file=config/examples/nni-nas-search-space-tf2.json
configFile: test/training_service/config/examples/classic-nas-tf2.yml
launchCommand: nnictl ss_gen --trial_command="python3 train.py --epochs 1" --trial_dir=../examples/nas/legacy/classic_nas-tf --file=training_service/config/examples/nni-nas-search-space-tf2.json
stopCommand:
experimentStatusCheck: False
trainingService: local
- name: classic-nas-tensorflow2
configFile: test/config/examples/classic-nas-tf2.yml
configFile: test/training_service/config/examples/classic-nas-tf2.yml
# remove search space file
stopCommand: nnictl stop
onExitCommand: python3 -c 'import os; os.remove("config/examples/nni-nas-search-space-tf2.json")'
onExitCommand: python3 -c 'import os; os.remove("training_service/config/examples/nni-nas-search-space-tf2.json")'
trainingService: local
#########################################################################
# nni features test
#########################################################################
- name: metrics-float
configFile: test/config/metrics_test/config.yml
configFile: test/training_service/config/metrics_test/config.yml
config:
maxTrialNum: 1
trialConcurrency: 1
......@@ -84,7 +84,7 @@ testCases:
expected_result_file: expected_metrics.json
- name: export-float
configFile: test/config/metrics_test/config.yml
configFile: test/training_service/config/metrics_test/config.yml
config:
maxTrialNum: 1
trialConcurrency: 1
......@@ -92,7 +92,7 @@ testCases:
class: ExportValidator
- name: metrics-dict
configFile: test/config/metrics_test/config_dict_metrics.yml
configFile: test/training_service/config/metrics_test/config_dict_metrics.yml
config:
maxTrialNum: 1
trialConcurrency: 1
......@@ -102,7 +102,7 @@ testCases:
expected_result_file: expected_metrics_dict.json
- name: export-dict
configFile: test/config/metrics_test/config_dict_metrics.yml
configFile: test/training_service/config/metrics_test/config_dict_metrics.yml
config:
maxTrialNum: 1
trialConcurrency: 1
......@@ -110,25 +110,25 @@ testCases:
class: ExportValidator
- name: foreground
configFile: test/config/examples/sklearn-regression.yml
launchCommand: python3 nni_test/nnitest/foreground.py --config $configFile --timeout 45
configFile: test/training_service/config/examples/sklearn-regression.yml
launchCommand: python3 training_service/nnitest/foreground.py --config $configFile --timeout 45
stopCommand:
experimentStatusCheck: False
platform: linux darwin
# Experiment resume test part 1
- name: nnictl-resume-1
configFile: test/config/examples/sklearn-regression.yml
configFile: test/training_service/config/examples/sklearn-regression.yml
setExperimentIdtoVar: $resumeExpId
# Experiment resume test part 2
- name: nnictl-resume-2
configFile: test/config/examples/sklearn-regression.yml
configFile: test/training_service/config/examples/sklearn-regression.yml
launchCommand: nnictl resume $resumeExpId
# Experiment view test
- name: nnictl-view
configFile: test/config/examples/sklearn-regression.yml
configFile: test/training_service/config/examples/sklearn-regression.yml
launchCommand: nnictl view $resumeExpId
experimentStatusCheck: False
......@@ -136,7 +136,7 @@ testCases:
# nni assessor test
#########################################################################
- name: assessor-curvefitting
configFile: test/config/assessors/curvefitting.yml
configFile: test/training_service/config/assessors/curvefitting.yml
- name: assessor-medianstop
configFile: test/config/assessors/medianstop.yml
configFile: test/training_service/config/assessors/medianstop.yml
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment