Skip to content

GitLab

Unverified Commit c7ca4510 authored by SparkSnail's avatar SparkSnail Committed by GitHub
Browse files

Merge pull request #171 from microsoft/master 

merge master
parents a030505a af89df8c
Show whitespace changes
Inline Side-by-side
Showing with 635 additions and 285 deletions
examples/trials/mnist-cascading-search-space/search_space.json
View file @ c7ca4510
{ {
"layer0":{"_type":"choice","_value":[ "layer0": {
"Empty", "_type": "choice",
["Conv", {"_type":"choice","_value":[2,3,5]}], "_value": [{
["Max_pool", {"_type":"choice","_value":[2,3,5]}], "_name": "Empty"
["Avg_pool", {"_type":"choice","_value":[2,3,5]}] },
]}, {
"layer1":{"_type":"choice","_value":[ "_name": "Conv",
"Empty", "kernel_size": {
["Conv", {"_type":"choice","_value":[2,3,5]}], "_type": "choice",
["Max_pool", {"_type":"choice","_value":[2,3,5]}], "_value": [1, 2, 3, 5]
["Avg_pool", {"_type":"choice","_value":[2,3,5]}] }
]}, },
"layer2":{"_type":"choice","_value":[ {
"Empty", "_name": "Max_pool",
["Conv", {"_type":"choice","_value":[2,3,5]}], "pooling_size": {
["Max_pool", {"_type":"choice","_value":[2,3,5]}], "_type": "choice",
["Avg_pool", {"_type":"choice","_value":[2,3,5]}] "_value": [2, 3, 5]
]}, }
"layer3":{"_type":"choice","_value":[ },
"Empty", {
["Conv", {"_type":"choice","_value":[2,3,5]}], "_name": "Avg_pool",
["Max_pool", {"_type":"choice","_value":[2,3,5]}], "pooling_size": {
["Avg_pool", {"_type":"choice","_value":[2,3,5]}] "_type": "choice",
]}, "_value": [2, 3, 5]
"layer4":{"_type":"choice","_value":[ }
"Empty", }
["Conv", {"_type":"choice","_value":[2,3,5]}], ]
["Max_pool", {"_type":"choice","_value":[2,3,5]}], },
["Avg_pool", {"_type":"choice","_value":[2,3,5]}] "layer1": {
]}, "_type": "choice",
"layer5":{"_type":"choice","_value":[ "_value": [{
"Empty", "_name": "Empty"
["Conv", {"_type":"choice","_value":[2,3,5]}], },
["Max_pool", {"_type":"choice","_value":[2,3,5]}], {
["Avg_pool", {"_type":"choice","_value":[2,3,5]}] "_name": "Conv",
]}, "kernel_size": {
"layer6":{"_type":"choice","_value":[ "_type": "choice",
"Empty", "_value": [1, 2, 3, 5]
["Conv", {"_type":"choice","_value":[2,3,5]}], }
["Max_pool", {"_type":"choice","_value":[2,3,5]}], },
["Avg_pool", {"_type":"choice","_value":[2,3,5]}] {
]}, "_name": "Max_pool",
"layer7":{"_type":"choice","_value":[ "pooling_size": {
"Empty", "_type": "choice",
["Conv", {"_type":"choice","_value":[2,3,5]}], "_value": [2, 3, 5]
["Max_pool", {"_type":"choice","_value":[2,3,5]}], }
["Avg_pool", {"_type":"choice","_value":[2,3,5]}] },
]}, {
"layer8":{"_type":"choice","_value":[ "_name": "Avg_pool",
"Empty", "pooling_size": {
["Conv", {"_type":"choice","_value":[2,3,5]}], "_type": "choice",
["Max_pool", {"_type":"choice","_value":[2,3,5]}], "_value": [2, 3, 5]
["Avg_pool", {"_type":"choice","_value":[2,3,5]}] }
]}, }
"layer9":{"_type":"choice","_value":[ ]
"Empty", },
["Conv", {"_type":"choice","_value":[2,3,5]}], "layer2": {
["Max_pool", {"_type":"choice","_value":[2,3,5]}], "_type": "choice",
["Avg_pool", {"_type":"choice","_value":[2,3,5]}] "_value": [{
]} "_name": "Empty"
},
{
"_name": "Conv",
"kernel_size": {
"_type": "choice",
"_value": [1, 2, 3, 5]
}
},
{
"_name": "Max_pool",
"pooling_size": {
"_type": "choice",
"_value": [2, 3, 5]
}
},
{
"_name": "Avg_pool",
"pooling_size": {
"_type": "choice",
"_value": [2, 3, 5]
}
}
]
},
"layer3": {
"_type": "choice",
"_value": [{
"_name": "Empty"
},
{
"_name": "Conv",
"kernel_size": {
"_type": "choice",
"_value": [1, 2, 3, 5]
}
},
{
"_name": "Max_pool",
"pooling_size": {
"_type": "choice",
"_value": [2, 3, 5]
}
},
{
"_name": "Avg_pool",
"pooling_size": {
"_type": "choice",
"_value": [2, 3, 5]
}
}
]
}
} }
\ No newline at end of file
src/nni_manager/core/nnimanager.ts
View file @ c7ca4510
...@@ -372,7 +372,7 @@ class NNIManager implements Manager { ...@@ -372,7 +372,7 @@ class NNIManager implements Manager {
private async periodicallyUpdateExecDuration(): Promise<void> { private async periodicallyUpdateExecDuration(): Promise<void> {
let count: number = 1; let count: number = 1;
while (this.status.status !== 'STOPPING' && this.status.status !== 'STOPPED') { while (!['ERROR', 'STOPPING', 'STOPPED'].includes(this.status.status)) {
await delay(1000 * 1); // 1 seconds await delay(1000 * 1); // 1 seconds
if (this.status.status === 'RUNNING') { if (this.status.status === 'RUNNING') {
this.experimentProfile.execDuration += 1; this.experimentProfile.execDuration += 1;
...@@ -461,7 +461,7 @@ class NNIManager implements Manager { ...@@ -461,7 +461,7 @@ class NNIManager implements Manager {
} }
let allFinishedTrialJobNum: number = this.currSubmittedTrialNum; let allFinishedTrialJobNum: number = this.currSubmittedTrialNum;
let waitSubmittedToFinish: number; let waitSubmittedToFinish: number;
while (this.status.status !== 'STOPPING' && this.status.status !== 'STOPPED') { while (!['ERROR', 'STOPPING', 'STOPPED'].includes(this.status.status)) {
const finishedTrialJobNum: number = await this.requestTrialJobsStatus(); const finishedTrialJobNum: number = await this.requestTrialJobsStatus();
allFinishedTrialJobNum += finishedTrialJobNum; allFinishedTrialJobNum += finishedTrialJobNum;
...@@ -671,7 +671,9 @@ class NNIManager implements Manager { ...@@ -671,7 +671,9 @@ class NNIManager implements Manager {
'ADD_HYPERPARAMETER', tunerCommand.trial_job_id, content, undefined); 'ADD_HYPERPARAMETER', tunerCommand.trial_job_id, content, undefined);
break; break;
case NO_MORE_TRIAL_JOBS: case NO_MORE_TRIAL_JOBS:
if (!['ERROR', 'STOPPING', 'STOPPED'].includes(this.status.status)) {
this.setStatus('TUNER_NO_MORE_TRIAL'); this.setStatus('TUNER_NO_MORE_TRIAL');
}
break; break;
case KILL_TRIAL_JOB: case KILL_TRIAL_JOB:
this.log.info(`cancelTrialJob: ${JSON.parse(content)}`); this.log.info(`cancelTrialJob: ${JSON.parse(content)}`);
......
src/nni_manager/training_service/local/gpuScheduler.ts
View file @ c7ca4510
...@@ -105,13 +105,17 @@ class GPUScheduler { ...@@ -105,13 +105,17 @@ class GPUScheduler {
} }
private async updateGPUSummary(): Promise<void> { private async updateGPUSummary(): Promise<void> {
const cmdresult: cpp.childProcessPromise.Result = let gpuMetricPath = path.join(this.gpuMetricCollectorScriptFolder, 'gpu_metrics');
await execTail(path.join(this.gpuMetricCollectorScriptFolder, 'gpu_metrics')); if (fs.existsSync(gpuMetricPath)) {
const cmdresult: cpp.childProcessPromise.Result = await execTail(gpuMetricPath);
if (cmdresult && cmdresult.stdout) { if (cmdresult && cmdresult.stdout) {
this.gpuSummary = <GPUSummary>JSON.parse(cmdresult.stdout); this.gpuSummary = <GPUSummary>JSON.parse(cmdresult.stdout);
} else { } else {
this.log.error('Could not get gpu metrics information!'); this.log.error('Could not get gpu metrics information!');
} }
} else{
this.log.warning('gpu_metrics file does not exist!')
}
} }
} }
......
src/sdk/pynni/nni/bohb_advisor/bohb_advisor.py
View file @ c7ca4510
...@@ -21,7 +21,6 @@ ...@@ -21,7 +21,6 @@
bohb_advisor.py bohb_advisor.py
''' '''
from enum import Enum, unique
import sys import sys
import math import math
import logging import logging
...@@ -32,7 +31,7 @@ import ConfigSpace.hyperparameters as CSH ...@@ -32,7 +31,7 @@ import ConfigSpace.hyperparameters as CSH
from nni.protocol import CommandType, send from nni.protocol import CommandType, send
from nni.msg_dispatcher_base import MsgDispatcherBase from nni.msg_dispatcher_base import MsgDispatcherBase
from nni.utils import extract_scalar_reward from nni.utils import OptimizeMode, extract_scalar_reward
from .config_generator import CG_BOHB from .config_generator import CG_BOHB
...@@ -42,12 +41,6 @@ _next_parameter_id = 0 ...@@ -42,12 +41,6 @@ _next_parameter_id = 0
_KEY = 'TRIAL_BUDGET' _KEY = 'TRIAL_BUDGET'
_epsilon = 1e-6 _epsilon = 1e-6
@unique
class OptimizeMode(Enum):
"""Optimize Mode class"""
Minimize = 'minimize'
Maximize = 'maximize'
def create_parameter_id(): def create_parameter_id():
"""Create an id """Create an id
......
src/sdk/pynni/nni/evolution_tuner/evolution_tuner.py
View file @ c7ca4510
...@@ -18,61 +18,34 @@ ...@@ -18,61 +18,34 @@
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
""" """
evolution_tuner.py including: evolution_tuner.py
class OptimizeMode
class Individual
class EvolutionTuner
""" """
import copy import copy
from enum import Enum, unique
import random import random
import numpy as np import numpy as np
from nni.tuner import Tuner from nni.tuner import Tuner
from nni.utils import extract_scalar_reward from nni.utils import NodeType, OptimizeMode, extract_scalar_reward, split_index
from .. import parameter_expressions
@unique
class OptimizeMode(Enum):
"""Optimize Mode class
if OptimizeMode is 'minimize', it means the tuner need to minimize the reward
that received from Trial.
if OptimizeMode is 'maximize', it means the tuner need to maximize the reward
that received from Trial.
"""
Minimize = 'minimize'
Maximize = 'maximize'
import nni.parameter_expressions as parameter_expressions
@unique
class NodeType(Enum):
"""Node Type class
"""
Root = 'root'
Type = '_type'
Value = '_value'
Index = '_index'
def json2space(x, oldy=None, name=NodeType.ROOT):
def json2space(x, oldy=None, name=NodeType.Root.value):
"""Change search space from json format to hyperopt format """Change search space from json format to hyperopt format
""" """
y = list() y = list()
if isinstance(x, dict): if isinstance(x, dict):
if NodeType.Type.value in x.keys(): if NodeType.TYPE in x.keys():
_type = x[NodeType.Type.value] _type = x[NodeType.TYPE]
name = name + '-' + _type name = name + '-' + _type
if _type == 'choice': if _type == 'choice':
if oldy != None: if oldy != None:
_index = oldy[NodeType.Index.value] _index = oldy[NodeType.INDEX]
y += json2space(x[NodeType.Value.value][_index], y += json2space(x[NodeType.VALUE][_index],
oldy[NodeType.Value.value], name=name+'[%d]' % _index) oldy[NodeType.VALUE], name=name+'[%d]' % _index)
else: else:
y += json2space(x[NodeType.Value.value], None, name=name) y += json2space(x[NodeType.VALUE], None, name=name)
y.append(name) y.append(name)
else: else:
for key in x.keys(): for key in x.keys():
...@@ -80,28 +53,28 @@ def json2space(x, oldy=None, name=NodeType.Root.value): ...@@ -80,28 +53,28 @@ def json2space(x, oldy=None, name=NodeType.Root.value):
None else None), name+"[%s]" % str(key)) None else None), name+"[%s]" % str(key))
elif isinstance(x, list): elif isinstance(x, list):
for i, x_i in enumerate(x): for i, x_i in enumerate(x):
if isinstance(x_i, dict):
if NodeType.NAME not in x_i.keys():
raise RuntimeError('\'_name\' key is not found in this nested search space.')
y += json2space(x_i, (oldy[i] if oldy != y += json2space(x_i, (oldy[i] if oldy !=
None else None), name+"[%d]" % i) None else None), name+"[%d]" % i)
else:
pass
return y return y
def json2parameter(x, is_rand, random_state, oldy=None, Rand=False, name=NodeType.ROOT):
def json2paramater(x, is_rand, random_state, oldy=None, Rand=False, name=NodeType.Root.value):
"""Json to pramaters. """Json to pramaters.
""" """
if isinstance(x, dict): if isinstance(x, dict):
if NodeType.Type.value in x.keys(): if NodeType.TYPE in x.keys():
_type = x[NodeType.Type.value] _type = x[NodeType.TYPE]
_value = x[NodeType.Value.value] _value = x[NodeType.VALUE]
name = name + '-' + _type name = name + '-' + _type
Rand |= is_rand[name] Rand |= is_rand[name]
if Rand is True: if Rand is True:
if _type == 'choice': if _type == 'choice':
_index = random_state.randint(len(_value)) _index = random_state.randint(len(_value))
y = { y = {
NodeType.Index.value: _index, NodeType.INDEX: _index,
NodeType.Value.value: json2paramater(x[NodeType.Value.value][_index], NodeType.VALUE: json2parameter(x[NodeType.VALUE][_index],
is_rand, is_rand,
random_state, random_state,
None, None,
...@@ -116,39 +89,20 @@ def json2paramater(x, is_rand, random_state, oldy=None, Rand=False, name=NodeTyp ...@@ -116,39 +89,20 @@ def json2paramater(x, is_rand, random_state, oldy=None, Rand=False, name=NodeTyp
else: else:
y = dict() y = dict()
for key in x.keys(): for key in x.keys():
y[key] = json2paramater(x[key], is_rand, random_state, oldy[key] y[key] = json2parameter(x[key], is_rand, random_state, oldy[key]
if oldy != None else None, Rand, name + "[%s]" % str(key)) if oldy != None else None, Rand, name + "[%s]" % str(key))
elif isinstance(x, list): elif isinstance(x, list):
y = list() y = list()
for i, x_i in enumerate(x): for i, x_i in enumerate(x):
y.append(json2paramater(x_i, is_rand, random_state, oldy[i] if isinstance(x_i, dict):
if NodeType.NAME not in x_i.keys():
raise RuntimeError('\'_name\' key is not found in this nested search space.')
y.append(json2parameter(x_i, is_rand, random_state, oldy[i]
if oldy != None else None, Rand, name + "[%d]" % i)) if oldy != None else None, Rand, name + "[%d]" % i))
else: else:
y = copy.deepcopy(x) y = copy.deepcopy(x)
return y return y
def _split_index(params):
"""Delete index information from params
Parameters
----------
params : dict
Returns
-------
result : dict
"""
result = {}
for key in params:
if isinstance(params[key], dict):
value = params[key]['_value']
else:
value = params[key]
result[key] = value
return result
class Individual(object): class Individual(object):
""" """
Indicidual class to store the indv info. Indicidual class to store the indv info.
...@@ -229,7 +183,7 @@ class EvolutionTuner(Tuner): ...@@ -229,7 +183,7 @@ class EvolutionTuner(Tuner):
for item in self.space: for item in self.space:
is_rand[item] = True is_rand[item] = True
for _ in range(self.population_size): for _ in range(self.population_size):
config = json2paramater( config = json2parameter(
self.searchspace_json, is_rand, self.random_state) self.searchspace_json, is_rand, self.random_state)
self.population.append(Individual(config=config)) self.population.append(Individual(config=config))
...@@ -267,14 +221,14 @@ class EvolutionTuner(Tuner): ...@@ -267,14 +221,14 @@ class EvolutionTuner(Tuner):
mutation_pos = space[random.randint(0, len(space)-1)] mutation_pos = space[random.randint(0, len(space)-1)]
for i in range(len(self.space)): for i in range(len(self.space)):
is_rand[self.space[i]] = (self.space[i] == mutation_pos) is_rand[self.space[i]] = (self.space[i] == mutation_pos)
config = json2paramater( config = json2parameter(
self.searchspace_json, is_rand, self.random_state, self.population[0].config) self.searchspace_json, is_rand, self.random_state, self.population[0].config)
self.population.pop(1) self.population.pop(1)
# remove "_index" from config and save params-id # remove "_index" from config and save params-id
total_config = config total_config = config
self.total_data[parameter_id] = total_config self.total_data[parameter_id] = total_config
config = _split_index(total_config) config = split_index(total_config)
return config return config
def receive_trial_result(self, parameter_id, parameters, value): def receive_trial_result(self, parameter_id, parameters, value):
......
src/sdk/pynni/nni/evolution_tuner/test_evolution_tuner.py 0 → 100644
View file @ c7ca4510
# Copyright (c) Microsoft Corporation
# All rights reserved.
#
# MIT License
#
# Permission is hereby granted, free of charge,
# to any person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and
# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
test_evolution_tuner.py
"""
import numpy as np
from unittest import TestCase, main
from nni.evolution_tuner.evolution_tuner import json2space, json2parameter
class EvolutionTunerTestCase(TestCase):
def test_json2space(self):
"""test for json2space
"""
json_search_space = {
"optimizer": {
"_type": "choice",
"_value": ["Adam", "SGD"]
},
"learning_rate": {
"_type": "choice",
"_value": [0.0001, 0.001, 0.002, 0.005, 0.01]
}
}
search_space_instance = json2space(json_search_space)
self.assertIn('root[optimizer]-choice', search_space_instance)
self.assertIn('root[learning_rate]-choice', search_space_instance)
def test_json2parameter(self):
"""test for json2parameter
"""
json_search_space = {
"optimizer":{
"_type":"choice","_value":["Adam", "SGD"]
},
"learning_rate":{
"_type":"choice",
"_value":[0.0001, 0.001, 0.002, 0.005, 0.01]
}
}
space = json2space(json_search_space)
random_state = np.random.RandomState()
is_rand = dict()
for item in space:
is_rand[item] = True
search_space_instance = json2parameter(json_search_space, is_rand, random_state)
self.assertIn(search_space_instance["optimizer"]["_index"], range(2))
self.assertIn(search_space_instance["optimizer"]["_value"], ["Adam", "SGD"])
self.assertIn(search_space_instance["learning_rate"]["_index"], range(5))
self.assertIn(search_space_instance["learning_rate"]["_value"], [0.0001, 0.001, 0.002, 0.005, 0.01])
if __name__ == '__main__':
main()
src/sdk/pynni/nni/gridsearch_tuner/gridsearch_tuner.py
View file @ c7ca4510
...@@ -56,7 +56,7 @@ class GridSearchTuner(Tuner): ...@@ -56,7 +56,7 @@ class GridSearchTuner(Tuner):
self.expanded_search_space = [] self.expanded_search_space = []
self.supplement_data = dict() self.supplement_data = dict()
def json2paramater(self, ss_spec): def json2parameter(self, ss_spec):
''' '''
generate all possible configs for hyperparameters from hyperparameter space. generate all possible configs for hyperparameters from hyperparameter space.
ss_spec: hyperparameter space ss_spec: hyperparameter space
...@@ -68,7 +68,7 @@ class GridSearchTuner(Tuner): ...@@ -68,7 +68,7 @@ class GridSearchTuner(Tuner):
chosen_params = list() chosen_params = list()
if _type == 'choice': if _type == 'choice':
for value in _value: for value in _value:
choice = self.json2paramater(value) choice = self.json2parameter(value)
if isinstance(choice, list): if isinstance(choice, list):
chosen_params.extend(choice) chosen_params.extend(choice)
else: else:
...@@ -78,12 +78,12 @@ class GridSearchTuner(Tuner): ...@@ -78,12 +78,12 @@ class GridSearchTuner(Tuner):
else: else:
chosen_params = dict() chosen_params = dict()
for key in ss_spec.keys(): for key in ss_spec.keys():
chosen_params[key] = self.json2paramater(ss_spec[key]) chosen_params[key] = self.json2parameter(ss_spec[key])
return self.expand_parameters(chosen_params) return self.expand_parameters(chosen_params)
elif isinstance(ss_spec, list): elif isinstance(ss_spec, list):
chosen_params = list() chosen_params = list()
for subspec in ss_spec[1:]: for subspec in ss_spec[1:]:
choice = self.json2paramater(subspec) choice = self.json2parameter(subspec)
if isinstance(choice, list): if isinstance(choice, list):
chosen_params.extend(choice) chosen_params.extend(choice)
else: else:
...@@ -135,7 +135,7 @@ class GridSearchTuner(Tuner): ...@@ -135,7 +135,7 @@ class GridSearchTuner(Tuner):
''' '''
Check if the search space is valid and expand it: only contains 'choice' type or other types beginnning with the letter 'q' Check if the search space is valid and expand it: only contains 'choice' type or other types beginnning with the letter 'q'
''' '''
self.expanded_search_space = self.json2paramater(search_space) self.expanded_search_space = self.json2parameter(search_space)
def generate_parameters(self, parameter_id): def generate_parameters(self, parameter_id):
self.count += 1 self.count += 1
......
src/sdk/pynni/nni/hyperband_advisor/hyperband_advisor.py
View file @ c7ca4510
...@@ -21,7 +21,6 @@ ...@@ -21,7 +21,6 @@
hyperband_advisor.py hyperband_advisor.py
""" """
from enum import Enum, unique
import sys import sys
import math import math
import copy import copy
...@@ -31,8 +30,9 @@ import json_tricks ...@@ -31,8 +30,9 @@ import json_tricks
from nni.protocol import CommandType, send from nni.protocol import CommandType, send
from nni.msg_dispatcher_base import MsgDispatcherBase from nni.msg_dispatcher_base import MsgDispatcherBase
from nni.utils import extract_scalar_reward from nni.common import init_logger
from .. import parameter_expressions from nni.utils import NodeType, OptimizeMode, extract_scalar_reward
import nni.parameter_expressions as parameter_expressions
_logger = logging.getLogger(__name__) _logger = logging.getLogger(__name__)
...@@ -40,11 +40,6 @@ _next_parameter_id = 0 ...@@ -40,11 +40,6 @@ _next_parameter_id = 0
_KEY = 'TRIAL_BUDGET' _KEY = 'TRIAL_BUDGET'
_epsilon = 1e-6 _epsilon = 1e-6
@unique
class OptimizeMode(Enum):
"""Oprimize Mode class"""
Minimize = 'minimize'
Maximize = 'maximize'
def create_parameter_id(): def create_parameter_id():
"""Create an id """Create an id
...@@ -82,7 +77,7 @@ def create_bracket_parameter_id(brackets_id, brackets_curr_decay, increased_id=- ...@@ -82,7 +77,7 @@ def create_bracket_parameter_id(brackets_id, brackets_curr_decay, increased_id=-
increased_id]) increased_id])
return params_id return params_id
def json2paramater(ss_spec, random_state): def json2parameter(ss_spec, random_state):
"""Randomly generate values for hyperparameters from hyperparameter space i.e., x. """Randomly generate values for hyperparameters from hyperparameter space i.e., x.
Parameters Parameters
...@@ -98,23 +93,23 @@ def json2paramater(ss_spec, random_state): ...@@ -98,23 +93,23 @@ def json2paramater(ss_spec, random_state):
Parameters in this experiment Parameters in this experiment
""" """
if isinstance(ss_spec, dict): if isinstance(ss_spec, dict):
if '_type' in ss_spec.keys(): if NodeType.TYPE in ss_spec.keys():
_type = ss_spec['_type'] _type = ss_spec[NodeType.TYPE]
_value = ss_spec['_value'] _value = ss_spec[NodeType.VALUE]
if _type == 'choice': if _type == 'choice':
_index = random_state.randint(len(_value)) _index = random_state.randint(len(_value))
chosen_params = json2paramater(ss_spec['_value'][_index], random_state) chosen_params = json2parameter(ss_spec[NodeType.VALUE][_index], random_state)
else: else:
chosen_params = eval('parameter_expressions.' + # pylint: disable=eval-used chosen_params = eval('parameter_expressions.' + # pylint: disable=eval-used
_type)(*(_value + [random_state])) _type)(*(_value + [random_state]))
else: else:
chosen_params = dict() chosen_params = dict()
for key in ss_spec.keys(): for key in ss_spec.keys():
chosen_params[key] = json2paramater(ss_spec[key], random_state) chosen_params[key] = json2parameter(ss_spec[key], random_state)
elif isinstance(ss_spec, list): elif isinstance(ss_spec, list):
chosen_params = list() chosen_params = list()
for _, subspec in enumerate(ss_spec): for _, subspec in enumerate(ss_spec):
chosen_params.append(json2paramater(subspec, random_state)) chosen_params.append(json2parameter(subspec, random_state))
else: else:
chosen_params = copy.deepcopy(ss_spec) chosen_params = copy.deepcopy(ss_spec)
return chosen_params return chosen_params
...@@ -246,7 +241,7 @@ class Bracket(): ...@@ -246,7 +241,7 @@ class Bracket():
hyperparameter_configs = dict() hyperparameter_configs = dict()
for _ in range(num): for _ in range(num):
params_id = create_bracket_parameter_id(self.bracket_id, self.i) params_id = create_bracket_parameter_id(self.bracket_id, self.i)
params = json2paramater(searchspace_json, random_state) params = json2parameter(searchspace_json, random_state)
params[_KEY] = r params[_KEY] = r
hyperparameter_configs[params_id] = params hyperparameter_configs[params_id] = params
self._record_hyper_configs(hyperparameter_configs) self._record_hyper_configs(hyperparameter_configs)
......
src/sdk/pynni/nni/hyperopt_tuner/hyperopt_tuner.py
View file @ c7ca4510
...@@ -17,39 +17,22 @@ ...@@ -17,39 +17,22 @@
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
''' """
hyperopt_tuner.py hyperopt_tuner.py
''' """
import copy import copy
import logging import logging
from enum import Enum, unique
import numpy as np
import hyperopt as hp import hyperopt as hp
import numpy as np
from nni.tuner import Tuner from nni.tuner import Tuner
from nni.utils import extract_scalar_reward from nni.utils import NodeType, OptimizeMode, extract_scalar_reward, split_index
logger = logging.getLogger('hyperopt_AutoML') logger = logging.getLogger('hyperopt_AutoML')
@unique def json2space(in_x, name=NodeType.ROOT):
class OptimizeMode(Enum):
"""
Optimize Mode including Minimize and Maximize
"""
Minimize = 'minimize'
Maximize = 'maximize'
ROOT = 'root'
TYPE = '_type'
VALUE = '_value'
INDEX = '_index'
def json2space(in_x, name=ROOT):
""" """
Change json to search space in hyperopt. Change json to search space in hyperopt.
...@@ -58,16 +41,16 @@ def json2space(in_x, name=ROOT): ...@@ -58,16 +41,16 @@ def json2space(in_x, name=ROOT):
in_x : dict/list/str/int/float in_x : dict/list/str/int/float
The part of json. The part of json.
name : str name : str
name could be ROOT, TYPE, VALUE or INDEX. name could be NodeType.ROOT, NodeType.TYPE, NodeType.VALUE or NodeType.INDEX, NodeType.NAME.
""" """
out_y = copy.deepcopy(in_x) out_y = copy.deepcopy(in_x)
if isinstance(in_x, dict): if isinstance(in_x, dict):
if TYPE in in_x.keys(): if NodeType.TYPE in in_x.keys():
_type = in_x[TYPE] _type = in_x[NodeType.TYPE]
name = name + '-' + _type name = name + '-' + _type
_value = json2space(in_x[VALUE], name=name) _value = json2space(in_x[NodeType.VALUE], name=name)
if _type == 'choice': if _type == 'choice':
out_y = eval('hp.hp.'+_type)(name, _value) out_y = eval('hp.hp.choice')(name, _value)
else: else:
if _type in ['loguniform', 'qloguniform']: if _type in ['loguniform', 'qloguniform']:
_value[:2] = np.log(_value[:2]) _value[:2] = np.log(_value[:2])
...@@ -75,70 +58,93 @@ def json2space(in_x, name=ROOT): ...@@ -75,70 +58,93 @@ def json2space(in_x, name=ROOT):
else: else:
out_y = dict() out_y = dict()
for key in in_x.keys(): for key in in_x.keys():
out_y[key] = json2space(in_x[key], name+'[%s]' % str(key)) out_y[key] = json2space(in_x[key], name + '[%s]' % str(key))
elif isinstance(in_x, list): elif isinstance(in_x, list):
out_y = list() out_y = list()
for i, x_i in enumerate(in_x): for i, x_i in enumerate(in_x):
out_y.append(json2space(x_i, name+'[%d]' % i)) if isinstance(x_i, dict):
else: if NodeType.NAME not in x_i.keys():
logger.info('in_x is not a dict or a list in json2space fuinction %s', str(in_x)) raise RuntimeError(
'\'_name\' key is not found in this nested search space.'
)
out_y.append(json2space(x_i, name + '[%d]' % i))
return out_y return out_y
def json2parameter(in_x, parameter, name=ROOT): def json2parameter(in_x, parameter, name=NodeType.ROOT):
""" """
Change json to parameters. Change json to parameters.
""" """
out_y = copy.deepcopy(in_x) out_y = copy.deepcopy(in_x)
if isinstance(in_x, dict): if isinstance(in_x, dict):
if TYPE in in_x.keys(): if NodeType.TYPE in in_x.keys():
_type = in_x[TYPE] _type = in_x[NodeType.TYPE]
name = name + '-' + _type name = name + '-' + _type
if _type == 'choice': if _type == 'choice':
_index = parameter[name] _index = parameter[name]
out_y = { out_y = {
INDEX: _index, NodeType.INDEX:
VALUE: json2parameter(in_x[VALUE][_index], parameter, name=name+'[%d]' % _index) _index,
NodeType.VALUE:
json2parameter(in_x[NodeType.VALUE][_index],
parameter,
name=name + '[%d]' % _index)
} }
else: else:
out_y = parameter[name] out_y = parameter[name]
else: else:
out_y = dict() out_y = dict()
for key in in_x.keys(): for key in in_x.keys():
out_y[key] = json2parameter( out_y[key] = json2parameter(in_x[key], parameter,
in_x[key], parameter, name + '[%s]' % str(key)) name + '[%s]' % str(key))
elif isinstance(in_x, list): elif isinstance(in_x, list):
out_y = list() out_y = list()
for i, x_i in enumerate(in_x): for i, x_i in enumerate(in_x):
if isinstance(x_i, dict):
if NodeType.NAME not in x_i.keys():
raise RuntimeError(
'\'_name\' key is not found in this nested search space.'
)
out_y.append(json2parameter(x_i, parameter, name + '[%d]' % i)) out_y.append(json2parameter(x_i, parameter, name + '[%d]' % i))
else:
logger.info('in_x is not a dict or a list in json2space fuinction %s', str(in_x))
return out_y return out_y
def json2vals(in_x, vals, out_y, name=ROOT): def json2vals(in_x, vals, out_y, name=NodeType.ROOT):
if isinstance(in_x, dict): if isinstance(in_x, dict):
if TYPE in in_x.keys(): if NodeType.TYPE in in_x.keys():
_type = in_x[TYPE] _type = in_x[NodeType.TYPE]
name = name + '-' + _type name = name + '-' + _type
try: try:
out_y[name] = vals[INDEX] out_y[name] = vals[NodeType.INDEX]
# TODO - catch exact Exception # TODO - catch exact Exception
except Exception: except Exception:
out_y[name] = vals out_y[name] = vals
if _type == 'choice': if _type == 'choice':
_index = vals[INDEX] _index = vals[NodeType.INDEX]
json2vals(in_x[VALUE][_index], vals[VALUE], json2vals(in_x[NodeType.VALUE][_index],
out_y, name=name + '[%d]' % _index) vals[NodeType.VALUE],
out_y,
name=name + '[%d]' % _index)
else: else:
for key in in_x.keys(): for key in in_x.keys():
json2vals(in_x[key], vals[key], out_y, name + '[%s]' % str(key)) json2vals(in_x[key], vals[key], out_y,
name + '[%s]' % str(key))
elif isinstance(in_x, list): elif isinstance(in_x, list):
for i, temp in enumerate(in_x): for i, temp in enumerate(in_x):
# nested json
if isinstance(temp, dict):
if NodeType.NAME not in temp.keys():
raise RuntimeError(
'\'_name\' key is not found in this nested search space.'
)
else:
json2vals(temp, vals[i], out_y, name + '[%d]' % i)
else:
json2vals(temp, vals[i], out_y, name + '[%d]' % i) json2vals(temp, vals[i], out_y, name + '[%d]' % i)
def _add_index(in_x, parameter): def _add_index(in_x, parameter):
""" """
change parameters in NNI format to parameters in hyperopt format(This function also support nested dict.). change parameters in NNI format to parameters in hyperopt format(This function also support nested dict.).
...@@ -156,41 +162,36 @@ def _add_index(in_x, parameter): ...@@ -156,41 +162,36 @@ def _add_index(in_x, parameter):
value_type = in_x[TYPE] value_type = in_x[TYPE]
value_format = in_x[VALUE] value_format = in_x[VALUE]
if value_type == "choice": if value_type == "choice":
choice_name = parameter[0] if isinstance(parameter, list) else parameter choice_name = parameter[0] if isinstance(parameter,
for pos, item in enumerate(value_format): # here value_format is a list list) else parameter
if isinstance(item, list): # this format is ["choice_key", format_dict] for pos, item in enumerate(
value_format): # here value_format is a list
if isinstance(
item,
list): # this format is ["choice_key", format_dict]
choice_key = item[0] choice_key = item[0]
choice_value_format = item[1] choice_value_format = item[1]
if choice_key == choice_name: if choice_key == choice_name:
return {INDEX: pos, VALUE: [choice_name, _add_index(choice_value_format, parameter[1])]} return {
INDEX:
pos,
VALUE: [
choice_name,
_add_index(choice_value_format, parameter[1])
]
}
elif choice_name == item: elif choice_name == item:
return {INDEX: pos, VALUE: item} return {INDEX: pos, VALUE: item}
else: else:
return parameter return parameter
def _split_index(params):
"""
Delete index infromation from params
"""
if isinstance(params, list):
return [params[0], _split_index(params[1])]
elif isinstance(params, dict):
if INDEX in params.keys():
return _split_index(params[VALUE])
result = dict()
for key in params:
result[key] = _split_index(params[key])
return result
else:
return params
class HyperoptTuner(Tuner): class HyperoptTuner(Tuner):
""" """
HyperoptTuner is a tuner which using hyperopt algorithm. HyperoptTuner is a tuner which using hyperopt algorithm.
""" """
def __init__(self, algorithm_name, optimize_mode = 'minimize'): def __init__(self, algorithm_name, optimize_mode='minimize'):
""" """
Parameters Parameters
---------- ----------
...@@ -234,11 +235,16 @@ class HyperoptTuner(Tuner): ...@@ -234,11 +235,16 @@ class HyperoptTuner(Tuner):
search_space_instance = json2space(self.json) search_space_instance = json2space(self.json)
rstate = np.random.RandomState() rstate = np.random.RandomState()
trials = hp.Trials() trials = hp.Trials()
domain = hp.Domain(None, search_space_instance, domain = hp.Domain(None,
search_space_instance,
pass_expr_memo_ctrl=None) pass_expr_memo_ctrl=None)
algorithm = self._choose_tuner(self.algorithm_name) algorithm = self._choose_tuner(self.algorithm_name)
self.rval = hp.FMinIter(algorithm, domain, trials, self.rval = hp.FMinIter(algorithm,
max_evals=-1, rstate=rstate, verbose=0) domain,
trials,
max_evals=-1,
rstate=rstate,
verbose=0)
self.rval.catch_eval_exceptions = False self.rval.catch_eval_exceptions = False
def generate_parameters(self, parameter_id): def generate_parameters(self, parameter_id):
...@@ -259,7 +265,7 @@ class HyperoptTuner(Tuner): ...@@ -259,7 +265,7 @@ class HyperoptTuner(Tuner):
# but it can cause deplicate parameter rarely # but it can cause deplicate parameter rarely
total_params = self.get_suggestion(random_search=True) total_params = self.get_suggestion(random_search=True)
self.total_data[parameter_id] = total_params self.total_data[parameter_id] = total_params
params = _split_index(total_params) params = split_index(total_params)
return params return params
def receive_trial_result(self, parameter_id, parameters, value): def receive_trial_result(self, parameter_id, parameters, value):
...@@ -300,7 +306,7 @@ class HyperoptTuner(Tuner): ...@@ -300,7 +306,7 @@ class HyperoptTuner(Tuner):
json2vals(self.json, vals, out_y) json2vals(self.json, vals, out_y)
vals = out_y vals = out_y
for key in domain.params: for key in domain.params:
if key in [VALUE, INDEX]: if key in [NodeType.VALUE, NodeType.INDEX]:
continue continue
if key not in vals or vals[key] is None or vals[key] == []: if key not in vals or vals[key] is None or vals[key] == []:
idxs[key] = vals[key] = [] idxs[key] = vals[key] = []
...@@ -308,17 +314,23 @@ class HyperoptTuner(Tuner): ...@@ -308,17 +314,23 @@ class HyperoptTuner(Tuner):
idxs[key] = [new_id] idxs[key] = [new_id]
vals[key] = [vals[key]] vals[key] = [vals[key]]
self.miscs_update_idxs_vals(rval_miscs, idxs, vals, self.miscs_update_idxs_vals(rval_miscs,
idxs,
vals,
idxs_map={new_id: new_id}, idxs_map={new_id: new_id},
assert_all_vals_used=False) assert_all_vals_used=False)
trial = trials.new_trial_docs([new_id], rval_specs, rval_results, rval_miscs)[0] trial = trials.new_trial_docs([new_id], rval_specs, rval_results,
rval_miscs)[0]
trial['result'] = {'loss': reward, 'status': 'ok'} trial['result'] = {'loss': reward, 'status': 'ok'}
trial['state'] = hp.JOB_STATE_DONE trial['state'] = hp.JOB_STATE_DONE
trials.insert_trial_docs([trial]) trials.insert_trial_docs([trial])
trials.refresh() trials.refresh()
def miscs_update_idxs_vals(self, miscs, idxs, vals, def miscs_update_idxs_vals(self,
miscs,
idxs,
vals,
assert_all_vals_used=True, assert_all_vals_used=True,
idxs_map=None): idxs_map=None):
""" """
...@@ -368,9 +380,10 @@ class HyperoptTuner(Tuner): ...@@ -368,9 +380,10 @@ class HyperoptTuner(Tuner):
algorithm = rval.algo algorithm = rval.algo
new_ids = rval.trials.new_trial_ids(1) new_ids = rval.trials.new_trial_ids(1)
rval.trials.refresh() rval.trials.refresh()
random_state = rval.rstate.randint(2**31-1) random_state = rval.rstate.randint(2**31 - 1)
if random_search: if random_search:
new_trials = hp.rand.suggest(new_ids, rval.domain, trials, random_state) new_trials = hp.rand.suggest(new_ids, rval.domain, trials,
random_state)
else: else:
new_trials = algorithm(new_ids, rval.domain, trials, random_state) new_trials = algorithm(new_ids, rval.domain, trials, random_state)
rval.trials.refresh() rval.trials.refresh()
...@@ -396,7 +409,8 @@ class HyperoptTuner(Tuner): ...@@ -396,7 +409,8 @@ class HyperoptTuner(Tuner):
""" """
_completed_num = 0 _completed_num = 0
for trial_info in data: for trial_info in data:
logger.info("Importing data, current processing progress %s / %s" %(_completed_num, len(data))) logger.info("Importing data, current processing progress %s / %s" %
(_completed_num, len(data)))
_completed_num += 1 _completed_num += 1
if self.algorithm_name == 'random_search': if self.algorithm_name == 'random_search':
return return
...@@ -405,10 +419,16 @@ class HyperoptTuner(Tuner): ...@@ -405,10 +419,16 @@ class HyperoptTuner(Tuner):
assert "value" in trial_info assert "value" in trial_info
_value = trial_info['value'] _value = trial_info['value']
if not _value: if not _value:
logger.info("Useless trial data, value is %s, skip this trial data." %_value) logger.info(
"Useless trial data, value is %s, skip this trial data." %
_value)
continue continue
self.supplement_data_num += 1 self.supplement_data_num += 1
_parameter_id = '_'.join(["ImportData", str(self.supplement_data_num)]) _parameter_id = '_'.join(
self.total_data[_parameter_id] = _add_index(in_x=self.json, parameter=_params) ["ImportData", str(self.supplement_data_num)])
self.receive_trial_result(parameter_id=_parameter_id, parameters=_params, value=_value) self.total_data[_parameter_id] = _add_index(in_x=self.json,
parameter=_params)
self.receive_trial_result(parameter_id=_parameter_id,
parameters=_params,
value=_value)
logger.info("Successfully import data to TPE/Anneal tuner.") logger.info("Successfully import data to TPE/Anneal tuner.")
src/sdk/pynni/nni/hyperopt_tuner/test_hyperopt_tuner.py 0 → 100644
View file @ c7ca4510
# Copyright (c) Microsoft Corporation
# All rights reserved.
#
# MIT License
#
# Permission is hereby granted, free of charge,
# to any person obtaining a copy of this software and associated
# documentation files (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and
# to permit persons to whom the Software is furnished to do so, subject to the following conditions:
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
# BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
test_hyperopt_tuner.py
"""
from unittest import TestCase, main
import hyperopt as hp
from nni.hyperopt_tuner.hyperopt_tuner import json2space, json2parameter, json2vals
class HyperoptTunerTestCase(TestCase):
def test_json2space(self):
"""test for json2space
"""
json_search_space = {
"optimizer": {
"_type": "choice",
"_value": ["Adam", "SGD"]
},
"learning_rate": {
"_type": "choice",
"_value": [0.0001, 0.001, 0.002, 0.005, 0.01]
}
}
search_space_instance = json2space(json_search_space)
self.assertIsInstance(search_space_instance["optimizer"],
hp.pyll.base.Apply)
self.assertIsInstance(search_space_instance["learning_rate"],
hp.pyll.base.Apply)
def test_json2parameter(self):
"""test for json2parameter
"""
json_search_space = {
"optimizer": {
"_type": "choice",
"_value": ["Adam", "SGD"]
},
"learning_rate": {
"_type": "choice",
"_value": [0.0001, 0.001, 0.002, 0.005, 0.01]
}
}
parameter = {
'root[learning_rate]-choice': 2,
'root[optimizer]-choice': 0
}
search_space_instance = json2parameter(json_search_space, parameter)
self.assertEqual(search_space_instance["optimizer"]["_index"], 0)
self.assertEqual(search_space_instance["optimizer"]["_value"], "Adam")
self.assertEqual(search_space_instance["learning_rate"]["_index"], 2)
self.assertEqual(search_space_instance["learning_rate"]["_value"], 0.002)
def test_json2vals(self):
"""test for json2vals
"""
json_search_space = {
"optimizer": {
"_type": "choice",
"_value": ["Adam", "SGD"]
},
"learning_rate": {
"_type": "choice",
"_value": [0.0001, 0.001, 0.002, 0.005, 0.01]
}
}
out_y = dict()
vals = {
'optimizer': {
'_index': 0,
'_value': 'Adam'
},
'learning_rate': {
'_index': 1,
'_value': 0.001
}
}
json2vals(json_search_space, vals, out_y)
self.assertEqual(out_y["root[optimizer]-choice"], 0)
self.assertEqual(out_y["root[learning_rate]-choice"], 1)
if __name__ == '__main__':
main()
src/sdk/pynni/nni/metis_tuner/metis_tuner.py
View file @ c7ca4510
...@@ -38,17 +38,10 @@ import nni.metis_tuner.Regression_GP.OutlierDetection as gp_outlier_detection ...@@ -38,17 +38,10 @@ import nni.metis_tuner.Regression_GP.OutlierDetection as gp_outlier_detection
import nni.metis_tuner.Regression_GP.Prediction as gp_prediction import nni.metis_tuner.Regression_GP.Prediction as gp_prediction
import nni.metis_tuner.Regression_GP.Selection as gp_selection import nni.metis_tuner.Regression_GP.Selection as gp_selection
from nni.tuner import Tuner from nni.tuner import Tuner
from nni.utils import extract_scalar_reward from nni.utils import OptimizeMode, extract_scalar_reward
logger = logging.getLogger("Metis_Tuner_AutoML") logger = logging.getLogger("Metis_Tuner_AutoML")
@unique
class OptimizeMode(Enum):
"""
Optimize Mode class
"""
Minimize = 'minimize'
Maximize = 'maximize'
NONE_TYPE = '' NONE_TYPE = ''
......
src/sdk/pynni/nni/networkmorphism_tuner/networkmorphism_tuner.py
View file @ c7ca4510
...@@ -23,10 +23,10 @@ import os ...@@ -23,10 +23,10 @@ import os
from nni.tuner import Tuner from nni.tuner import Tuner
from nni.utils import extract_scalar_reward from nni.utils import OptimizeMode, extract_scalar_reward
from nni.networkmorphism_tuner.bayesian import BayesianOptimizer from nni.networkmorphism_tuner.bayesian import BayesianOptimizer
from nni.networkmorphism_tuner.nn import CnnGenerator, MlpGenerator from nni.networkmorphism_tuner.nn import CnnGenerator, MlpGenerator
from nni.networkmorphism_tuner.utils import Constant, OptimizeMode from nni.networkmorphism_tuner.utils import Constant
from nni.networkmorphism_tuner.graph import graph_to_json, json_to_graph from nni.networkmorphism_tuner.graph import graph_to_json, json_to_graph
......
src/sdk/pynni/nni/networkmorphism_tuner/utils.py
View file @ c7ca4510
...@@ -18,16 +18,6 @@ ...@@ -18,16 +18,6 @@
# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# ================================================================================================== # ==================================================================================================
from enum import Enum, unique
@unique
class OptimizeMode(Enum):
"""
Oprimize Mode class
"""
Minimize = "minimize"
Maximize = "maximize"
class Constant: class Constant:
'''Constant for the Tuner. '''Constant for the Tuner.
......
src/sdk/pynni/nni/smac_tuner/smac_tuner.py
View file @ c7ca4510
...@@ -22,7 +22,7 @@ smac_tuner.py ...@@ -22,7 +22,7 @@ smac_tuner.py
""" """
from nni.tuner import Tuner from nni.tuner import Tuner
from nni.utils import extract_scalar_reward from nni.utils import OptimizeMode, extract_scalar_reward
import sys import sys
import logging import logging
...@@ -37,11 +37,6 @@ from smac.facade.smac_facade import SMAC ...@@ -37,11 +37,6 @@ from smac.facade.smac_facade import SMAC
from smac.facade.roar_facade import ROAR from smac.facade.roar_facade import ROAR
from smac.facade.epils_facade import EPILS from smac.facade.epils_facade import EPILS
@unique
class OptimizeMode(Enum):
"""Oprimize Mode class"""
Minimize = 'minimize'
Maximize = 'maximize'
class SMACTuner(Tuner): class SMACTuner(Tuner):
""" """
......
src/sdk/pynni/nni/utils.py
View file @ c7ca4510
...@@ -17,11 +17,54 @@ ...@@ -17,11 +17,54 @@
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT # DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# ================================================================================================== # ==================================================================================================
"""
utils.py
"""
import os import os
from enum import Enum, unique
from .common import init_logger from .common import init_logger
from .env_vars import dispatcher_env_vars from .env_vars import dispatcher_env_vars
@unique
class OptimizeMode(Enum):
"""Optimize Mode class
if OptimizeMode is 'minimize', it means the tuner need to minimize the reward
that received from Trial.
if OptimizeMode is 'maximize', it means the tuner need to maximize the reward
that received from Trial.
"""
Minimize = 'minimize'
Maximize = 'maximize'
class NodeType:
"""Node Type class
"""
ROOT = 'root'
TYPE = '_type'
VALUE = '_value'
INDEX = '_index'
NAME = '_name'
def split_index(params):
"""
Delete index infromation from params
"""
if isinstance(params, dict):
if NodeType.INDEX in params.keys():
return split_index(params[NodeType.VALUE])
result = {}
for key in params:
result[key] = split_index(params[key])
return result
else:
return params
def extract_scalar_reward(value, scalar_key='default'): def extract_scalar_reward(value, scalar_key='default'):
""" """
Extract scalar reward from trial result. Extract scalar reward from trial result.
......
src/sdk/pynni/tests/test_utils.py 0 → 100644
View file @ c7ca4510
# Copyright (c) Microsoft Corporation. All rights reserved.
#
# MIT License
#
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and
# associated documentation files (the "Software"), to deal in the Software without restriction,
# including without limitation the rights to use, copy, modify, merge, publish, distribute,
# sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all copies or
# substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT
# NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
# DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT
# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# ==================================================================================================
from unittest import TestCase, main
import nni
from nni.utils import split_index
class UtilsTestCase(TestCase):
def test_split_index_normal(self):
"""test for normal search space
"""
normal__params_with_index = {
"dropout_rate": {
"_index" : 1,
"_value" : 0.9
},
"hidden_size": {
"_index" : 1,
"_value" : 512
}
}
normal__params= {
"dropout_rate": 0.9,
"hidden_size": 512
}
params = split_index(normal__params_with_index)
self.assertEqual(params, normal__params)
def test_split_index_nested(self):
"""test for nested search space
"""
nested_params_with_index = {
"layer0": {
"_name": "Avg_pool",
"pooling_size":{
"_index" : 1,
"_value" : 2
}
},
"layer1": {
"_name": "Empty"
},
"layer2": {
"_name": "Max_pool",
"pooling_size": {
"_index" : 2,
"_value" : 3
}
},
"layer3": {
"_name": "Conv",
"kernel_size": {
"_index" : 3,
"_value" : 5
},
"output_filters": {
"_index" : 3,
"_value" : 64
}
}
}
nested_params = {
"layer0": {
"_name": "Avg_pool",
"pooling_size": 2
},
"layer1": {
"_name": "Empty"
},
"layer2": {
"_name": "Max_pool",
"pooling_size": 3
},
"layer3": {
"_name": "Conv",
"kernel_size": 5,
"output_filters": 64
}
}
params = split_index(nested_params_with_index)
self.assertEqual(params, nested_params)
if __name__ == '__main__':
main()
\ No newline at end of file
test/config_test/examples/mnist-cascading-search-space.test.yml 0 → 100644
View file @ c7ca4510
authorName: nni
experimentName: default_test
maxExecDuration: 5m
maxTrialNum: 4
trialConcurrency: 2
searchSpacePath: ../../../examples/trials/mnist-cascading-search-space/search_space.json
tuner:
#choice: TPE, Random, Anneal, Evolution
builtinTunerName: TPE
assessor:
builtinAssessorName: Medianstop
classArgs:
optimize_mode: maximize
trial:
codeDir: ../../../examples/trials/mnist-cascading-search-space
command: python3 mnist.py --batch_num 100
gpuNum: 0
useAnnotation: false
multiPhase: false
multiThread: false
trainingServicePlatform: local
tools/README.md
View file @ c7ca4510
...@@ -54,4 +54,4 @@ python >= 3.5 ...@@ -54,4 +54,4 @@ python >= 3.5
please reference to the [NNI CTL document]. please reference to the [NNI CTL document].
[NNI CTL document]: ../docs/en_US/NNICTLDOC.md [NNI CTL document]: ../docs/en_US/Nnictl.md
tools/nni_annotation/search_space_generator.py
View file @ c7ca4510
...@@ -21,6 +21,7 @@ ...@@ -21,6 +21,7 @@
import ast import ast
import astor import astor
import numbers
# pylint: disable=unidiomatic-typecheck # pylint: disable=unidiomatic-typecheck
...@@ -87,8 +88,9 @@ class SearchSpaceGenerator(ast.NodeTransformer): ...@@ -87,8 +88,9 @@ class SearchSpaceGenerator(ast.NodeTransformer):
args = [key.n if type(key) is ast.Num else key.s for key in node.args[0].keys] args = [key.n if type(key) is ast.Num else key.s for key in node.args[0].keys]
else: else:
# arguments of other functions must be literal number # arguments of other functions must be literal number
assert all(type(arg) is ast.Num for arg in node.args), 'Smart parameter\'s arguments must be number literals' assert all(isinstance(ast.literal_eval(astor.to_source(arg)), numbers.Real) for arg in node.args), \
args = [arg.n for arg in node.args] 'Smart parameter\'s arguments must be number literals'
args = [ast.literal_eval(astor.to_source(arg)) for arg in node.args]
key = self.module_name + '/' + name + '/' + func key = self.module_name + '/' + name + '/' + func
# store key in ast.Call # store key in ast.Call
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment