integration_tests.yml 9.29 KB
Newer Older
chicm-ms's avatar
chicm-ms committed
1
2

defaultTestCaseConfig:
3
  launchCommand: nnictl create --config $configFile --debug
chicm-ms's avatar
chicm-ms committed
4
5
6
  stopCommand: nnictl stop
  experimentStatusCheck: True
  platform: linux darwin win32
chicm-ms's avatar
chicm-ms committed
7
  trainingService: all
chicm-ms's avatar
chicm-ms committed
8
9
10
11
12
13
14

testCases:
#######################################################################
# nni examples test
#######################################################################
- name: sklearn-classification
  # test case config yml file relative to nni source code directory
15
  configFile: test/training_service/config/examples/sklearn-classification.yml
chicm-ms's avatar
chicm-ms committed
16
17
18
19
20
21

  # test case specific config, the content of configFile will be overrided
  # by config section
  config:

  # validator is called after experiment is done
22
  # validator class needs to be implemented in training_service/nnitest/validators.py
chicm-ms's avatar
chicm-ms committed
23
24
25
  validator:

  # launch command, default launch command is 'nnictl create --config $configFile'
26
  launchCommand: nnictl create --config $configFile --debug
chicm-ms's avatar
chicm-ms committed
27
28
29
30
31
32
33
34
35
36

  # stop command, default stop command is 'nnictl stop', empty means no stop command
  stopCommand: nnictl stop

  # set experiment ID into variable, variable name should start with $, such as $expId
  setExperimentIdtoVar: $expId

  # check status of experiment before calling validator
  experimentStatusCheck: True

37
- name: shared-storage-remote-azureblob
38
  configFile: test/training_service/config/sharedstorage_test/config_sharedstorage_remote_azureblob.yml
39
40
41
42
43
44
45
  config:
    sharedStorage:
      localMountPoint: /tmp/nnimount/testlocalrootpath
      remoteMountPoint: /tmp/nnimount/testremoterootpath
      storageAccountName: nennistorage
      storageAccountKey: $(azureblob_token_test)
      containerName: sharedstorage
46
  trainingService: remote
47
48
49
50
51
  validator:
    class: FileExistValidator
    kwargs:
      rootpath: /tmp/nnimount/testlocalrootpath

J-shang's avatar
J-shang committed
52
- name: sklearn-regression
53
  configFile: test/training_service/config/examples/sklearn-regression.yml
J-shang's avatar
J-shang committed
54
55
56

# mount point in local may late to umount when stop experiment in shared-storage-remote-azureblob,
# so keep two shared storage tests away from each other.
57
- name: shared-storage-remote-nfs
58
  configFile: test/training_service/config/sharedstorage_test/config_sharedstorage_remote_nfs.yml
59
60
61
62
63
  trainingService: remote
  validator:
    class: FileExistValidator
    kwargs:
      rootpath: /tmp/nnimount/testlocalrootpath
64

liuzhe-lz's avatar
liuzhe-lz committed
65
- name: mnist-tensorflow
66
  configFile: test/training_service/config/examples/mnist-tfv2.yml
67
  config:
SparkSnail's avatar
SparkSnail committed
68
    maxExecDuration: 10m # This example will use longger time in remote mode, set max_duration to 10m to avoid timeout error.
69
70
    maxTrialNum: 1
    trialConcurrency: 1
liuzhe-lz's avatar
liuzhe-lz committed
71
  trainingService: local remote # FIXME: timeout on pai, looks like tensorflow failed to link CUDA
chicm-ms's avatar
chicm-ms committed
72

73
- name: mnist-pytorch-local
74
  configFile: test/training_service/config/examples/mnist-pytorch.yml
75
76
  # download data first, to prevent concurrent issue.
  launchCommand: python3 ../examples/trials/mnist-pytorch/mnist.py --epochs 1 --batch_num 0 --data_dir ../examples/trials/mnist-pytorch/data && nnictl create --config $configFile --debug
77
78
  trainingService: local

Ni Hao's avatar
Ni Hao committed
79
- name: mnist-pytorch-local-gpu
80
  configFile: test/training_service/config/examples/mnist-pytorch-gpu.yml
Ni Hao's avatar
Ni Hao committed
81
82
83
84
  # download data first, to prevent concurrent issue.
  launchCommand: python3 ../examples/trials/mnist-pytorch/mnist.py --epochs 1 --batch_num 0 --data_dir ../examples/trials/mnist-pytorch/data && nnictl create --config $configFile --debug
  trainingService: local

85
- name: mnist-pytorch-non-local
86
  configFile: test/training_service/config/examples/mnist-pytorch.yml
87
88
89
  # download data first, to prevent concurrent issue.
  launchCommand: nnictl create --config $configFile --debug
  trainingService: remote pai kubeflow frameworkcontroller dlts
chicm-ms's avatar
chicm-ms committed
90

liuzhe-lz's avatar
liuzhe-lz committed
91
92
# TODO: move this and following commented test cases to pytorch or tf2
#- name: mnist-annotation
93
#  configFile: test/training_service/config/examples/mnist-annotation.yml
liuzhe-lz's avatar
liuzhe-lz committed
94
95
96
#  config:
#    maxTrialNum: 1
#    trialConcurrency: 1
chicm-ms's avatar
chicm-ms committed
97
98

- name: cifar10-pytorch
99
  configFile: test/training_service/config/examples/cifar10-pytorch.yml
chicm-ms's avatar
chicm-ms committed
100
101
102
103
104
105
106
107
108
109
  config:
    # this example downloads large pretrained model weights
    # test 1 trial to save time
    maxExecDuration: 10m
    maxTrialNum: 1
    trialConcurrency: 1
    trial:
      command: python3 main.py --epochs 1 --batches 1
      gpuNum: 0

SparkSnail's avatar
SparkSnail committed
110
- name: cifar10-pytorch-adl
111
  configFile: test/training_service/config/examples/cifar10-pytorch-adl.yml
SparkSnail's avatar
SparkSnail committed
112
113
  trainingService: adl

liuzhe-lz's avatar
liuzhe-lz committed
114
#- name: nested-ss
115
#  configFile: test/training_service/config/examples/mnist-nested-search-space.yml
chicm-ms's avatar
chicm-ms committed
116

117
- name: classic-nas-gen-ss
118
119
  configFile: test/training_service/config/examples/classic-nas-pytorch.yml
  launchCommand: nnictl ss_gen --trial_command="python3 mnist.py --epochs 1" --trial_dir=../examples/nas/legacy/classic_nas --file=training_service/config/examples/nni-nas-search-space.json
120
121
  stopCommand:
  experimentStatusCheck: False
chicm-ms's avatar
chicm-ms committed
122
  trainingService: local
123
124

- name: classic-nas-pytorch
125
  configFile: test/training_service/config/examples/classic-nas-pytorch.yml
126
127
  # remove search space file
  stopCommand: nnictl stop
128
  onExitCommand: python3 -c "import os; os.remove('training_service/config/examples/nni-nas-search-space.json')"
chicm-ms's avatar
chicm-ms committed
129
  trainingService: local
chicm-ms's avatar
chicm-ms committed
130
131
132
133
134

#########################################################################
# nni features test
#########################################################################
- name: metrics-float
135
  configFile: test/training_service/config/metrics_test/config.yml
chicm-ms's avatar
chicm-ms committed
136
137
138
139
140
141
142
143
  config:
    maxTrialNum: 1
    trialConcurrency: 1
  validator:
    class: MetricsValidator
    kwargs:
      expected_result_file: expected_metrics.json

Yan Ni's avatar
Yan Ni committed
144
- name: export-float
145
  configFile: test/training_service/config/metrics_test/config.yml
Yan Ni's avatar
Yan Ni committed
146
147
148
149
150
151
  config:
    maxTrialNum: 1
    trialConcurrency: 1
  validator:
    class: ExportValidator 

chicm-ms's avatar
chicm-ms committed
152
- name: metrics-dict
153
  configFile: test/training_service/config/metrics_test/config_dict_metrics.yml
chicm-ms's avatar
chicm-ms committed
154
155
156
157
158
159
160
  config:
    maxTrialNum: 1
    trialConcurrency: 1
  validator:
    class: MetricsValidator
    kwargs:
      expected_result_file: expected_metrics_dict.json
chicm-ms's avatar
chicm-ms committed
161

Yan Ni's avatar
Yan Ni committed
162
- name: export-dict
163
  configFile: test/training_service/config/metrics_test/config_dict_metrics.yml
Yan Ni's avatar
Yan Ni committed
164
165
166
167
168
169
  config:
    maxTrialNum: 1
    trialConcurrency: 1
  validator:
    class: ExportValidator 

170
- name: experiment-import
171
  configFile: test/training_service/config/nnictl_experiment/sklearn-classification.yml
172
173
174
  validator:
    class: ImportValidator
    kwargs:
175
      import_data_file_path: training_service/config/nnictl_experiment/test_import.json
176

chicm-ms's avatar
chicm-ms committed
177
- name: foreground
178
179
  configFile: test/training_service/config/examples/sklearn-regression.yml
  launchCommand: python3 training_service/nnitest/foreground.py --config $configFile --timeout 45
chicm-ms's avatar
chicm-ms committed
180
181
182
183
184
185
  stopCommand:
  experimentStatusCheck: False
  platform: linux darwin

# Experiment resume test part 1
- name: nnictl-resume-1
186
  configFile: test/training_service/config/examples/sklearn-regression.yml
chicm-ms's avatar
chicm-ms committed
187
  setExperimentIdtoVar: $resumeExpId
188
  # for subfolder in codedir test
189
  launchCommand: python3 -c "import os; os.makedirs('../examples/trials/sklearn/regression/subfolder', exist_ok=True); open('../examples/trials/sklearn/regression/subfolder/subfile', 'a').close()" && nnictl create --config $configFile --debug
chicm-ms's avatar
chicm-ms committed
190
191
192

# Experiment resume test part 2
- name: nnictl-resume-2
193
  configFile: test/training_service/config/examples/sklearn-regression.yml
chicm-ms's avatar
chicm-ms committed
194
195
196
197
  launchCommand: nnictl resume $resumeExpId

# Experiment view test
- name: nnictl-view
198
  configFile: test/training_service/config/examples/sklearn-regression.yml
chicm-ms's avatar
chicm-ms committed
199
200
201
202
203
204
205
206
  launchCommand: nnictl view $resumeExpId
  experimentStatusCheck: False


#########################################################################
# nni assessor test
#########################################################################
- name: assessor-curvefitting
207
  configFile: test/training_service/config/assessors/curvefitting.yml
chicm-ms's avatar
chicm-ms committed
208
209

- name: assessor-medianstop
210
  configFile: test/training_service/config/assessors/medianstop.yml
chicm-ms's avatar
chicm-ms committed
211
212
213
214

#########################################################################
# nni tuners test
#########################################################################
liuzhe-lz's avatar
liuzhe-lz committed
215
#- name: tuner-annel
216
#  configFile: test/training_service/config/tuners/anneal.yml
chicm-ms's avatar
chicm-ms committed
217

liuzhe-lz's avatar
liuzhe-lz committed
218
#- name: tuner-evolution
219
#  configFile: test/training_service/config/tuners/evolution.yml
chicm-ms's avatar
chicm-ms committed
220

liuzhe-lz's avatar
liuzhe-lz committed
221
#- name: tuner-random
222
#  configFile: test/training_service/config/tuners/random.yml
chicm-ms's avatar
chicm-ms committed
223

liuzhe-lz's avatar
liuzhe-lz committed
224
#- name: tuner-smac
225
#  configFile: test/training_service/config/tuners/smac.yml
liuzhe-lz's avatar
liuzhe-lz committed
226
#  platform: linux darwin
chicm-ms's avatar
chicm-ms committed
227

liuzhe-lz's avatar
liuzhe-lz committed
228
#- name: tuner-tpe
229
#  configFile: test/training_service/config/tuners/tpe.yml
chicm-ms's avatar
chicm-ms committed
230

liuzhe-lz's avatar
liuzhe-lz committed
231
#- name: tuner-batch
232
#  configFile: test/training_service/config/tuners/batch.yml
chicm-ms's avatar
chicm-ms committed
233

liuzhe-lz's avatar
liuzhe-lz committed
234
#- name: tuner-bohb
235
#  configFile: test/training_service/config/tuners/bohb.yml
liuzhe-lz's avatar
liuzhe-lz committed
236
#  platform: linux darwin
chicm-ms's avatar
chicm-ms committed
237

liuzhe-lz's avatar
liuzhe-lz committed
238
#- name: tuner-gp
239
#  configFile: test/training_service/config/tuners/gp.yml
chicm-ms's avatar
chicm-ms committed
240

liuzhe-lz's avatar
liuzhe-lz committed
241
#- name: tuner-grid
242
#  configFile: test/training_service/config/tuners/gridsearch.yml
chicm-ms's avatar
chicm-ms committed
243

liuzhe-lz's avatar
liuzhe-lz committed
244
#- name: tuner-hyperband
245
#  configFile: test/training_service/config/tuners/hyperband.yml
chicm-ms's avatar
chicm-ms committed
246

liuzhe-lz's avatar
liuzhe-lz committed
247
#- name: tuner-metis
248
#  configFile: test/training_service/config/tuners/metis.yml
249

250
- name: tuner-regularized_evolution
251
  configFile: test/training_service/config/tuners/regularized_evolution_tuner.yml
252

253
254
255
256
#########################################################################
# nni customized-tuners test
#########################################################################
- name: customized-tuners-demotuner
257
  configFile: test/training_service/config/customized_tuners/demotuner-sklearn-classification.yml