Skip to content

GitLab

Switch branch/tag
Find file Normal viewHistoryPermalink
integration_tests.yml 5.49 KB
Newer Older
chicm-ms's avatar
Refactor integration tests (#2190)  
chicm-ms committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195 

defaultTestCaseConfig:
  launchCommand: nnictl create --config $configFile
  stopCommand: nnictl stop
  experimentStatusCheck: True
  platform: linux darwin win32

testCases:
#######################################################################
# nni examples test
#######################################################################
- name: sklearn-classification
  # test case config yml file relative to nni source code directory
  configFile: test/config/examples/sklearn-classification.yml

  # test case specific config, the content of configFile will be overrided
  # by config section
  config:

  # validator is called after experiment is done
  # validator class needs to be implemented in nni_test/nnitest/validators.py
  validator:

  # launch command, default launch command is 'nnictl create --config $configFile'
  launchCommand: nnictl create --config $configFile

  # stop command, default stop command is 'nnictl stop', empty means no stop command
  stopCommand: nnictl stop

  # set experiment ID into variable, variable name should start with $, such as $expId
  setExperimentIdtoVar: $expId

  # check status of experiment before calling validator
  experimentStatusCheck: True

- name: sklearn-regression
  configFile: test/config/examples/sklearn-regression.yml

- name: mnist-tfv1
  configFile: test/config/examples/mnist-tfv1.yml

- name: mnist-keras
  configFile: test/config/examples/mnist-keras.yml

- name: mnist-pytorch
  configFile: test/config/examples/mnist-pytorch.yml

- name: mnist-annotation
  configFile: test/config/examples/mnist-annotation.yml

- name: cifar10-pytorch
  configFile: test/config/examples/cifar10-pytorch.yml
  config:
    # this example downloads large pretrained model weights
    # test 1 trial to save time
    maxExecDuration: 10m
    maxTrialNum: 1
    trialConcurrency: 1
    trial:
      command: python3 main.py --epochs 1 --batches 1
      gpuNum: 0

- name: nested-ss
  configFile: test/config/examples/mnist-nested-search-space.yml


#########################################################################
# nni features test
#########################################################################
- name: metrics-float
  configFile: test/config/metrics_test/config.yml
  config:
    maxTrialNum: 1
    trialConcurrency: 1
  validator:
    class: MetricsValidator
    kwargs:
      expected_result_file: expected_metrics.json

# to be enabled
#- name: metrics-dict
#  configFile: test/config/metrics_test/config_dict_metrics.yml
#  config:
#    maxTrialNum: 1
#    trialConcurrency: 1
#  validator:
#    class: MetricsValidator
#    kwargs:
#      expected_result_file: expected_metrics_dict.json

- name: nnicli
  configFile: test/config/examples/sklearn-regression.yml
  config:
    maxTrialNum: 4
    trialConcurrency: 4
  launchCommand: python3 -c 'import nnicli as nc; nc.start_nni("$configFile")'
  stopCommand: python3 -c 'import nnicli as nc; nc.stop_nni()'
  validator:
    class: NnicliValidator
  platform: linux darwin

- name: foreground
  configFile: test/config/examples/sklearn-regression.yml
  launchCommand: python3 nni_test/nnitest/foreground.py --config $configFile --timeout 45
  stopCommand:
  experimentStatusCheck: False
  platform: linux darwin

# Experiment resume test part 1
- name: nnictl-resume-1
  configFile: test/config/examples/sklearn-regression.yml
  setExperimentIdtoVar: $resumeExpId

# Experiment resume test part 2
- name: nnictl-resume-2
  configFile: test/config/examples/sklearn-regression.yml
  launchCommand: nnictl resume $resumeExpId

# Experiment view test
- name: nnictl-view
  configFile: test/config/examples/sklearn-regression.yml
  launchCommand: nnictl view $resumeExpId
  experimentStatusCheck: False

- name: multi-thread
  configFile: test/config/multi_thread/config.yml

- name: multi-phase-batch
  configFile: test/config/multi_phase/batch.yml
  config:
    # for batch tuner, maxTrialNum can not exceed length of search space
    maxTrialNum: 2
    trialConcurrency: 2

- name: multi-phase-evolution
  configFile: test/config/multi_phase/evolution.yml

- name: multi-phase-grid
  configFile: test/config/multi_phase/grid.yml
  config:
    maxTrialNum: 2
    trialConcurrency: 2

- name: multi-phase-metis
  configFile: test/config/multi_phase/metis.yml

- name: multi-phase-tpe
  configFile: test/config/multi_phase/tpe.yml

#########################################################################
# nni assessor test
#########################################################################
- name: assessor-curvefitting
  configFile: test/config/assessors/curvefitting.yml

- name: assessor-medianstop
  configFile: test/config/assessors/medianstop.yml

#########################################################################
# nni tuners test
#########################################################################
- name: tuner-annel
  configFile: test/config/tuners/anneal.yml

- name: tuner-evolution
  configFile: test/config/tuners/evolution.yml

- name: tuner-random
  configFile: test/config/tuners/random.yml

- name: tuner-smac
  configFile: test/config/tuners/smac.yml
  platform: linux darwin

- name: tuner-tpe
  configFile: test/config/tuners/tpe.yml

- name: tuner-batch
  configFile: test/config/tuners/batch.yml

- name: tuner-bohb
  configFile: test/config/tuners/bohb.yml
  platform: linux darwin

- name: tuner-gp
  configFile: test/config/tuners/gp.yml

- name: tuner-grid
  configFile: test/config/tuners/gridsearch.yml

- name: tuner-hyperband
  configFile: test/config/tuners/hyperband.yml

- name: tuner-metis
  configFile: test/config/tuners/metis.yml