integration_tests_config_v2.yml 5.28 KB
Newer Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14

defaultTestCaseConfig:
  launchCommand: nnictl create --config $configFile --debug
  stopCommand: nnictl stop
  experimentStatusCheck: True
  platform: linux darwin win32
  trainingService: all

testCases:
#######################################################################
# nni examples test
#######################################################################
- name: sklearn-classification
  # test case config yml file relative to nni source code directory
15
  configFile: test/training_service/config/examples/sklearn-classification-v2.yml
16
17

- name: sklearn-regression
18
  configFile: test/training_service/config/examples/sklearn-regression-v2.yml
19
20

- name: mnist-tensorflow
21
  configFile: test/training_service/config/examples/mnist-tfv2-v2.yml
22
  trainingService: local remote
23
24

- name: mnist-pytorch-local
25
  configFile: test/training_service/config/examples/mnist-pytorch-v2.yml
26
27
28
29
30
  # download data first, to prevent concurrent issue.
  launchCommand: python3 ../examples/trials/mnist-pytorch/mnist.py --epochs 1 --batch_num 0 --data_dir ../examples/trials/mnist-pytorch/data && nnictl create --config $configFile --debug
  trainingService: local

- name: mnist-pytorch-non-local
31
  configFile: test/training_service/config/examples/mnist-pytorch-v2.yml
32
33
34
  trainingService: remote pai kubeflow frameworkcontroller dlts hybrid

- name: cifar10-pytorch
35
  configFile: test/training_service/config/examples/cifar10-pytorch-v2.yml
36
37

- name: cifar10-pytorch-adl
38
  configFile: test/training_service/config/examples/cifar10-pytorch-adl.yml
39
40
41
  trainingService: adl

- name: classic-nas-gen-ss
42
43
  configFile: test/training_service/config/examples/classic-nas-pytorch-v2.yml
  launchCommand: nnictl ss_gen --trial_command="python3 mnist.py --epochs 1" --trial_dir=../examples/nas/legacy/classic_nas --file=training_service/config/examples/nni-nas-search-space.json
44
45
46
47
48
  stopCommand:
  experimentStatusCheck: False
  trainingService: local

- name: classic-nas-pytorch
49
  configFile: test/training_service/config/examples/classic-nas-pytorch-v2.yml
50
51
  # remove search space file
  stopCommand: nnictl stop
52
  onExitCommand: python3 -c "import os; os.remove('training_service/examples/nni-nas-search-space.json')"
53
54
55
56
57
58
  trainingService: local

#########################################################################
# nni features test
#########################################################################
- name: metrics-float
59
  configFile: test/training_service/config/metrics_test/config-v2.yml
60
61
62
63
64
65
  validator:
    class: MetricsValidator
    kwargs:
      expected_result_file: expected_metrics.json

- name: export-float
66
  configFile: test/training_service/config/metrics_test/config-v2.yml
67
68
69
70
  validator:
    class: ExportValidator 

- name: metrics-dict
71
  configFile: test/training_service/config/metrics_test/config_dict_metrics-v2.yml
72
73
74
75
76
77
  validator:
    class: MetricsValidator
    kwargs:
      expected_result_file: expected_metrics_dict.json

- name: export-dict
78
  configFile: test/training_service/config/metrics_test/config_dict_metrics-v2.yml
79
80
81
82
  validator:
    class: ExportValidator 

- name: experiment-import
83
  configFile: test/training_service/config/nnictl_experiment/sklearn-classification-v2.yml
84
85
86
  validator:
    class: ImportValidator
    kwargs:
87
      import_data_file_path: training_service/config/nnictl_experiment/test_import.json
88
89

- name: foreground
90
91
  configFile: test/training_service/config/examples/sklearn-regression-v2.yml
  launchCommand: python3 training_service/nnitest/foreground.py --config $configFile --timeout 45
92
93
94
95
96
97
  stopCommand:
  experimentStatusCheck: False
  platform: linux darwin

# Experiment resume test part 1
- name: nnictl-resume-1
98
  configFile: test/training_service/config/examples/sklearn-regression-v2.yml
99
100
101
102
103
104
  setExperimentIdtoVar: $resumeExpId
  # for subfolder in codedir test
  launchCommand: python3 -c "import os; os.makedirs('../examples/trials/sklearn/regression/subfolder', exist_ok=True); open('../examples/trials/sklearn/regression/subfolder/subfile', 'a').close()" && nnictl create --config $configFile --debug

# Experiment resume test part 2
- name: nnictl-resume-2
105
  configFile: test/training_service/config/examples/sklearn-regression-v2.yml
106
107
108
109
  launchCommand: nnictl resume $resumeExpId

# Experiment view test
- name: nnictl-view
110
  configFile: test/training_service/config/examples/sklearn-regression-v2.yml
111
112
113
114
115
116
117
118
  launchCommand: nnictl view $resumeExpId
  experimentStatusCheck: False


#########################################################################
# nni assessor test
#########################################################################
- name: assessor-curvefitting
119
  configFile: test/training_service/config/assessors/curvefitting-v2.yml
120
121

- name: assessor-medianstop
122
  configFile: test/training_service/config/assessors/medianstop-v2.yml
123
124
125
126
127

#########################################################################
# nni tuners test
#########################################################################
- name: tuner-regularized_evolution
128
  configFile: test/training_service/config/tuners/regularized_evolution_tuner-v2.yml
129
130
131
132
133

#########################################################################
# nni customized-tuners test
#########################################################################
- name: customized-tuners-demotuner
134
  configFile: test/training_service/config/customized_tuners/demotuner-sklearn-classification-v2.yml
135