"git@developer.sourcefind.cn:OpenDAS/nni.git" did not exist on "60e1e01f625aa63905e1a43e33876d246141d375"
Commit 7242098e authored by ShufanHuang's avatar ShufanHuang Committed by chicm-ms
Browse files

Improve the performance of the curve fitting assessor (#717)

Improve the performance of the curve fitting assessor 
parent 263498de
...@@ -63,6 +63,7 @@ It is applicable in a wide range of performance curves, thus, can be used in var ...@@ -63,6 +63,7 @@ It is applicable in a wide range of performance curves, thus, can be used in var
* **optimize_mode** (*maximize or minimize, optional, default = maximize*) - If 'maximize', assessor will **stop** the trial with smaller expectation. If 'minimize', assessor will **stop** the trial with larger expectation. * **optimize_mode** (*maximize or minimize, optional, default = maximize*) - If 'maximize', assessor will **stop** the trial with smaller expectation. If 'minimize', assessor will **stop** the trial with larger expectation.
* **start_step** (*int, optional, default = 6*) - A trial is determined to be stopped or not, we start to predict only after receiving start_step number of reported intermediate results. * **start_step** (*int, optional, default = 6*) - A trial is determined to be stopped or not, we start to predict only after receiving start_step number of reported intermediate results.
* **threshold** (*float, optional, default = 0.95*) - The threshold that we decide to early stop the worse performance curve. For example: if threshold = 0.95, optimize_mode = maximize, best performance in the history is 0.9, then we will stop the trial which predict value is lower than 0.95 * 0.9 = 0.855. * **threshold** (*float, optional, default = 0.95*) - The threshold that we decide to early stop the worse performance curve. For example: if threshold = 0.95, optimize_mode = maximize, best performance in the history is 0.9, then we will stop the trial which predict value is lower than 0.95 * 0.9 = 0.855.
* **gap** (*int, optional, default = 1*) - The gap interval between Assesor judgements. For example: if gap = 2, start_step = 6, then we will assess the result when we get 6, 8, 10, 12...intermedian result.
**Usage example:** **Usage example:**
...@@ -75,4 +76,5 @@ assessor: ...@@ -75,4 +76,5 @@ assessor:
optimize_mode: maximize optimize_mode: maximize
start_step: 6 start_step: 6
threshold: 0.95 threshold: 0.95
gap: 1
``` ```
\ No newline at end of file
...@@ -57,6 +57,10 @@ assessor: ...@@ -57,6 +57,10 @@ assessor:
* The default value of threshold is 0.95. * The default value of threshold is 0.95.
# Kindly reminds that if you choose minimize mode, please adjust the value of threshold >= 1.0 (e.g threshold=1.1) # Kindly reminds that if you choose minimize mode, please adjust the value of threshold >= 1.0 (e.g threshold=1.1)
threshold: 0.95 threshold: 0.95
# (optional) The gap interval between Assesor judgements.
# For example: if gap = 2, start_step = 6, then we will assess the result when we get 6, 8, 10, 12...intermedian result.
* The default value of gap is 1.
gap: 1
``` ```
## 3. File Structure ## 3. File Structure
......
...@@ -16,6 +16,7 @@ ...@@ -16,6 +16,7 @@
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
import logging import logging
import datetime
from nni.assessor import Assessor, AssessResult from nni.assessor import Assessor, AssessResult
from .model_factory import CurveModel from .model_factory import CurveModel
...@@ -37,7 +38,7 @@ class CurvefittingAssessor(Assessor): ...@@ -37,7 +38,7 @@ class CurvefittingAssessor(Assessor):
threshold: float threshold: float
The threshold that we decide to early stop the worse performance curve. The threshold that we decide to early stop the worse performance curve.
""" """
def __init__(self, epoch_num=20, optimize_mode='maximize', start_step=6, threshold=0.95): def __init__(self, epoch_num=20, optimize_mode='maximize', start_step=6, threshold=0.95, gap=1):
if start_step <= 0: if start_step <= 0:
logger.warning('It\'s recommended to set start_step to a positive number') logger.warning('It\'s recommended to set start_step to a positive number')
# Record the target position we predict # Record the target position we predict
...@@ -54,6 +55,10 @@ class CurvefittingAssessor(Assessor): ...@@ -54,6 +55,10 @@ class CurvefittingAssessor(Assessor):
self.start_step = start_step self.start_step = start_step
# Record the compared threshold # Record the compared threshold
self.threshold = threshold self.threshold = threshold
# Record the number of gap
self.gap = gap
# Record the number of times of judgments
self.judgment_num = 0
# Record the best performance # Record the best performance
self.set_best_performance = False self.set_best_performance = False
self.completed_best_performance = None self.completed_best_performance = None
...@@ -100,14 +105,20 @@ class CurvefittingAssessor(Assessor): ...@@ -100,14 +105,20 @@ class CurvefittingAssessor(Assessor):
Exception Exception
unrecognize exception in curvefitting_assessor unrecognize exception in curvefitting_assessor
""" """
self.trial_job_id = trial_job_id
self.trial_history = trial_history self.trial_history = trial_history
if not self.set_best_performance:
return AssessResult.Good
curr_step = len(trial_history) curr_step = len(trial_history)
if curr_step < self.start_step: if curr_step < self.start_step:
return AssessResult.Good return AssessResult.Good
if not self.set_best_performance: if (curr_step - self.start_step) // self.gap <= self.judgment_num:
return AssessResult.Good return AssessResult.Good
self.judgment_num = (curr_step - self.start_step) // self.gap
try: try:
start_time = datetime.datetime.now()
# Predict the final result
curvemodel = CurveModel(self.target_pos) curvemodel = CurveModel(self.target_pos)
predict_y = curvemodel.predict(trial_history) predict_y = curvemodel.predict(trial_history)
logger.info('Prediction done. Trial job id = ', trial_job_id, '. Predict value = ', predict_y) logger.info('Prediction done. Trial job id = ', trial_job_id, '. Predict value = ', predict_y)
...@@ -115,6 +126,11 @@ class CurvefittingAssessor(Assessor): ...@@ -115,6 +126,11 @@ class CurvefittingAssessor(Assessor):
logger.info('wait for more information to predict precisely') logger.info('wait for more information to predict precisely')
return AssessResult.Good return AssessResult.Good
standard_performance = self.completed_best_performance * self.threshold standard_performance = self.completed_best_performance * self.threshold
end_time = datetime.datetime.now()
if (end_time - start_time).seconds > 60:
logger.warning('Curve Fitting Assessor Runtime Exceeds 60s, Trial Id = ', self.trial_job_id, 'Trial History = ', self.trial_history)
if self.higher_better: if self.higher_better:
if predict_y > standard_performance: if predict_y > standard_performance:
return AssessResult.Good return AssessResult.Good
......
...@@ -22,8 +22,6 @@ from .curvefunctions import * ...@@ -22,8 +22,6 @@ from .curvefunctions import *
# Number of curve functions we prepared, more details can be found in "curvefunctions.py" # Number of curve functions we prepared, more details can be found in "curvefunctions.py"
NUM_OF_FUNCTIONS = 12 NUM_OF_FUNCTIONS = 12
# Maximum number of iterations when fitting the curve optimal parameters
MAXFEV = 1000000
# Number of simulation time when we do MCMC sampling # Number of simulation time when we do MCMC sampling
NUM_OF_SIMULATION_TIME = 20 NUM_OF_SIMULATION_TIME = 20
# Number of samples we select when we do MCMC sampling # Number of samples we select when we do MCMC sampling
...@@ -65,17 +63,18 @@ class CurveModel(object): ...@@ -65,17 +63,18 @@ class CurveModel(object):
for i in range(NUM_OF_FUNCTIONS): for i in range(NUM_OF_FUNCTIONS):
model = curve_combination_models[i] model = curve_combination_models[i]
try: try:
# The maximum number of iterations to fit is 100*(N+1), where N is the number of elements in `x0`.
if model_para_num[model] == 2: if model_para_num[model] == 2:
a, b = optimize.curve_fit(all_models[model], x, y, maxfev=MAXFEV)[0] a, b = optimize.curve_fit(all_models[model], x, y)[0]
model_para[model][0] = a model_para[model][0] = a
model_para[model][1] = b model_para[model][1] = b
elif model_para_num[model] == 3: elif model_para_num[model] == 3:
a, b, c = optimize.curve_fit(all_models[model], x, y, maxfev=MAXFEV)[0] a, b, c = optimize.curve_fit(all_models[model], x, y)[0]
model_para[model][0] = a model_para[model][0] = a
model_para[model][1] = b model_para[model][1] = b
model_para[model][2] = c model_para[model][2] = c
elif model_para_num[model] == 4: elif model_para_num[model] == 4:
a, b, c, d = optimize.curve_fit(all_models[model], x, y, maxfev=MAXFEV)[0] a, b, c, d = optimize.curve_fit(all_models[model], x, y)[0]
model_para[model][0] = a model_para[model][0] = a
model_para[model][1] = b model_para[model][1] = b
model_para[model][2] = c model_para[model][2] = c
...@@ -110,7 +109,8 @@ class CurveModel(object): ...@@ -110,7 +109,8 @@ class CurveModel(object):
std = np.std(predict_data) std = np.std(predict_data)
for model in tmp_model: for model in tmp_model:
y = self.predict_y(model, self.target_pos) y = self.predict_y(model, self.target_pos)
if y < median + 3 * std and y > median - 3 * std: epsilon = self.point_num / 10 * std
if y < median + epsilon and y > median - epsilon:
self.effective_model.append(model) self.effective_model.append(model)
self.effective_model_num = len(self.effective_model) self.effective_model_num = len(self.effective_model)
logger.info('List of effective model: ', self.effective_model) logger.info('List of effective model: ', self.effective_model)
......
...@@ -100,7 +100,8 @@ Optional('assessor'): Or({ ...@@ -100,7 +100,8 @@ Optional('assessor'): Or({
'epoch_num': And(int, lambda x: 0 <= x <= 9999), 'epoch_num': And(int, lambda x: 0 <= x <= 9999),
Optional('optimize_mode'): Or('maximize', 'minimize'), Optional('optimize_mode'): Or('maximize', 'minimize'),
Optional('start_step'): And(int, lambda x: 0 <= x <= 9999), Optional('start_step'): And(int, lambda x: 0 <= x <= 9999),
Optional('threshold'): And(float, lambda x: 0.0 <= x <= 9999.0) Optional('threshold'): And(float, lambda x: 0.0 <= x <= 9999.0),
Optional('gap'): And(int, lambda x: 1 <= x <= 9999)
}, },
Optional('gpuNum'): And(int, lambda x: 0 <= x <= 99999) Optional('gpuNum'): And(int, lambda x: 0 <= x <= 99999)
},{ },{
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment