"3rdParty/InstallRBuild.sh" did not exist on "52bca8131d28631279571c0c3b2093fb6af1de29"
nnimanager.ts 37 KB
Newer Older
liuzhe-lz's avatar
liuzhe-lz committed
1
2
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
Deshui Yu's avatar
Deshui Yu committed
3
4
5
6

'use strict';

import * as assert from 'assert';
chicm-ms's avatar
chicm-ms committed
7
import { ChildProcess, StdioOptions } from 'child_process';
Deshui Yu's avatar
Deshui Yu committed
8
9
10
import { Deferred } from 'ts-deferred';
import * as component from '../common/component';
import { DataStore, MetricDataRecord, MetricType, TrialJobInfo } from '../common/datastore';
11
import { NNIError } from '../common/errors';
12
import { getExperimentId, getDispatcherPipe } from '../common/experimentStartupInfo';
Deshui Yu's avatar
Deshui Yu committed
13
14
import { getLogger, Logger } from '../common/log';
import {
chicm-ms's avatar
chicm-ms committed
15
    ExperimentParams, ExperimentProfile, Manager, ExperimentStatus,
16
    NNIManagerStatus, ProfileUpdateType, TrialJobStatistics
Deshui Yu's avatar
Deshui Yu committed
17
} from '../common/manager';
18
import { ExperimentManager } from '../common/experimentManager';
Deshui Yu's avatar
Deshui Yu committed
19
import {
20
    TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric, TrialJobStatus, LogType
Deshui Yu's avatar
Deshui Yu committed
21
} from '../common/trainingService';
22
import { delay, getCheckpointDir, getExperimentRootDir, getLogDir, getMsgDispatcherCommand, mkDirP, getTunerProc, getLogLevel, isAlive, killPid } from '../common/utils';
Deshui Yu's avatar
Deshui Yu committed
23
import {
chicm-ms's avatar
chicm-ms committed
24
    INITIALIZE, INITIALIZED, KILL_TRIAL_JOB, NEW_TRIAL_JOB, NO_MORE_TRIAL_JOBS, PING,
25
    REPORT_METRIC_DATA, REQUEST_TRIAL_JOBS, SEND_TRIAL_JOB_PARAMETER, TERMINATE, TRIAL_END, UPDATE_SEARCH_SPACE, IMPORT_DATA
Deshui Yu's avatar
Deshui Yu committed
26
} from './commands';
27
import { createDispatcherInterface, createDispatcherPipeInterface, IpcInterface } from './ipcInterface';
Deshui Yu's avatar
Deshui Yu committed
28
29

/**
chicm-ms's avatar
chicm-ms committed
30
 * NNIManager which implements Manager interface
Deshui Yu's avatar
Deshui Yu committed
31
32
33
 */
class NNIManager implements Manager {
    private trainingService: TrainingService;
34
    private dispatcher: IpcInterface | undefined;
35
    private experimentManager: ExperimentManager;
36
    private currSubmittedTrialNum: number;  // need to be recovered
QuanluZhang's avatar
QuanluZhang committed
37
    private trialConcurrencyChange: number; // >0: increase, <0: decrease
Deshui Yu's avatar
Deshui Yu committed
38
39
40
    private log: Logger;
    private dataStore: DataStore;
    private experimentProfile: ExperimentProfile;
41
    private dispatcherPid: number;
42
    private status: NNIManagerStatus;
43
    private waitingTrials: TrialJobApplicationForm[];
QuanluZhang's avatar
QuanluZhang committed
44
    private trialJobs: Map<string, TrialJobDetail>;
45
    private trialDataForTuner: string;
SparkSnail's avatar
SparkSnail committed
46
    private readonly: boolean;
47

48
    private trialJobMetricListener: (metric: TrialJobMetric) => void;
49

Deshui Yu's avatar
Deshui Yu committed
50
51
    constructor() {
        this.currSubmittedTrialNum = 0;
QuanluZhang's avatar
QuanluZhang committed
52
        this.trialConcurrencyChange = 0;
Deshui Yu's avatar
Deshui Yu committed
53
        this.trainingService = component.get(TrainingService);
54
        this.experimentManager = component.get(ExperimentManager);
Deshui Yu's avatar
Deshui Yu committed
55
        assert(this.trainingService);
56
        this.dispatcherPid = 0;
QuanluZhang's avatar
QuanluZhang committed
57
58
        this.waitingTrials = [];
        this.trialJobs = new Map<string, TrialJobDetail>();
59
        this.trialDataForTuner = '';
SparkSnail's avatar
SparkSnail committed
60
        this.readonly = false;
Deshui Yu's avatar
Deshui Yu committed
61
62
63

        this.log = getLogger();
        this.dataStore = component.get(DataStore);
64
65
66
67
        this.experimentProfile = this.createEmptyExperimentProfile();
        this.status = {
            status: 'INITIALIZED',
            errors: []
Deshui Yu's avatar
Deshui Yu committed
68
        };
chicm-ms's avatar
chicm-ms committed
69
        this.trialJobMetricListener = (metric: TrialJobMetric): void => {
70
71
72
73
            this.onTrialJobMetrics(metric).catch((err: Error) => {
                this.criticalError(NNIError.FromError(err, 'Job metrics error: '));
            });
        };
74
75
76
77
78

        const pipe = getDispatcherPipe();
        if (pipe !== null) {
            this.dispatcher = createDispatcherPipeInterface(pipe);
        }
Deshui Yu's avatar
Deshui Yu committed
79
80
81
    }

    public updateExperimentProfile(experimentProfile: ExperimentProfile, updateType: ProfileUpdateType): Promise<void> {
SparkSnail's avatar
SparkSnail committed
82
83
84
        if (this.readonly) {
            return Promise.reject(new Error('Error: can not update experiment profile in readonly mode!'));
        }
Deshui Yu's avatar
Deshui Yu committed
85
86
87
88
89
90
91
92
93
94
        switch (updateType) {
            case 'TRIAL_CONCURRENCY':
                this.updateTrialConcurrency(experimentProfile.params.trialConcurrency);
                break;
            case 'MAX_EXEC_DURATION':
                this.updateMaxExecDuration(experimentProfile.params.maxExecDuration);
                break;
            case 'SEARCH_SPACE':
                this.updateSearchSpace(experimentProfile.params.searchSpace);
                break;
QuanluZhang's avatar
QuanluZhang committed
95
96
97
            case 'MAX_TRIAL_NUM':
                this.updateMaxTrialNum(experimentProfile.params.maxTrialNum);
                break;
Deshui Yu's avatar
Deshui Yu committed
98
99
100
101
102
103
104
            default:
                throw new Error('Error: unrecognized updateType');
        }

        return this.storeExperimentProfile();
    }

105
    public importData(data: string): Promise<void> {
SparkSnail's avatar
SparkSnail committed
106
107
108
        if (this.readonly) {
            return Promise.reject(new Error('Error: can not import data in readonly mode!'));
        }
109
110
111
112
113
114
115
116
117
118
        if (this.dispatcher === undefined) {
            return Promise.reject(
                new Error('tuner has not been setup')
            );
        }
        this.dispatcher.sendCommand(IMPORT_DATA, data);

        return this.dataStore.storeTrialJobEvent('IMPORT_DATA', '', data);
    }

119
120
121
122
    public getImportedData(): Promise<string[]> {
        return this.dataStore.getImportedData();
    }

123
124
125
126
    public async exportData(): Promise<string> {
        return this.dataStore.exportTrialHpConfigs();
    }

127
    public addCustomizedTrialJob(hyperParams: string): Promise<number> {
SparkSnail's avatar
SparkSnail committed
128
129
130
        if (this.readonly) {
            return Promise.reject(new Error('Error: can not add customized trial job in readonly mode!'));
        }
Deshui Yu's avatar
Deshui Yu committed
131
        if (this.currSubmittedTrialNum >= this.experimentProfile.params.maxTrialNum) {
132
            return Promise.reject(new Error('reach maxTrialNum'));
Deshui Yu's avatar
Deshui Yu committed
133
        }
134
135
136

        // TODO: NNI manager should not peek tuner's internal protocol, let's refactor this later
        const packedParameter = {
chicm-ms's avatar
chicm-ms committed
137
138
            parameter_id: null, // eslint-disable-line @typescript-eslint/camelcase
            parameter_source: 'customized', // eslint-disable-line @typescript-eslint/camelcase
139
140
141
142
143
144
145
146
147
148
149
            parameters: JSON.parse(hyperParams)
        }

        const form: TrialJobApplicationForm = {
            sequenceId: this.experimentProfile.nextSequenceId++,
            hyperParameters: {
                value: JSON.stringify(packedParameter),
                index: 0
            }
        };
        this.waitingTrials.push(form);
Deshui Yu's avatar
Deshui Yu committed
150
151

        // trial id has not been generated yet, thus use '' instead
152
153
154
        this.dataStore.storeTrialJobEvent('ADD_CUSTOMIZED', '', hyperParams);

        return Promise.resolve(form.sequenceId);
Deshui Yu's avatar
Deshui Yu committed
155
156
157
    }

    public async cancelTrialJobByUser(trialJobId: string): Promise<void> {
SparkSnail's avatar
SparkSnail committed
158
159
160
        if (this.readonly) {
            return Promise.reject(new Error('Error: can not cancel trial job in readonly mode!'));
        }
chicm-ms's avatar
chicm-ms committed
161
        this.log.info(`User cancelTrialJob: ${trialJobId}`);
Deshui Yu's avatar
Deshui Yu committed
162
163
164
165
166
        await this.trainingService.cancelTrialJob(trialJobId);
        await this.dataStore.storeTrialJobEvent('USER_TO_CANCEL', trialJobId, '');
    }

    public async startExperiment(expParams: ExperimentParams): Promise<string> {
chicm-ms's avatar
chicm-ms committed
167
        this.log.info(`Starting experiment: ${this.experimentProfile.id}`);
Deshui Yu's avatar
Deshui Yu committed
168
169
170
        this.experimentProfile.params = expParams;
        await this.storeExperimentProfile();
        this.log.debug('Setup tuner...');
171

172
        // Set up multiphase config
173
        if (expParams.multiPhase && this.trainingService.isMultiPhaseJobSupported) {
174
175
            this.trainingService.setClusterMetadata('multiPhase', expParams.multiPhase.toString());
        }
176
177
178
179
        // Set up versionCheck config
        if (expParams.versionCheck !== undefined) {
            this.trainingService.setClusterMetadata('version_check', expParams.versionCheck.toString());
        }
SparkSnail's avatar
SparkSnail committed
180
181
182
183
        // Set up logCollection config
        if (expParams.logCollection !== undefined) {
            this.trainingService.setClusterMetadata('log_collection', expParams.logCollection.toString());
        }
184

chicm-ms's avatar
chicm-ms committed
185
        const dispatcherCommand: string = getMsgDispatcherCommand(expParams);
186
        this.log.debug(`dispatcher command: ${dispatcherCommand}`);
QuanluZhang's avatar
QuanluZhang committed
187
        const checkpointDir: string = await this.createCheckpointDir();
Deshui Yu's avatar
Deshui Yu committed
188
        this.setupTuner(
189
190
            dispatcherCommand,
            undefined,
Deshui Yu's avatar
Deshui Yu committed
191
            'start',
QuanluZhang's avatar
QuanluZhang committed
192
            checkpointDir);
Deshui Yu's avatar
Deshui Yu committed
193

194
        this.experimentProfile.startTime = Date.now();
chicm-ms's avatar
chicm-ms committed
195
        this.setStatus('RUNNING');
Deshui Yu's avatar
Deshui Yu committed
196
        await this.storeExperimentProfile();
197
198
        this.run().catch((err: Error) => {
            this.criticalError(err);
Deshui Yu's avatar
Deshui Yu committed
199
        });
200

Deshui Yu's avatar
Deshui Yu committed
201
202
203
        return this.experimentProfile.id;
    }

SparkSnail's avatar
SparkSnail committed
204
    public async resumeExperiment(readonly: boolean): Promise<void> {
chicm-ms's avatar
chicm-ms committed
205
        this.log.info(`Resuming experiment: ${this.experimentProfile.id}`);
Deshui Yu's avatar
Deshui Yu committed
206
207
208
        //Fetch back the experiment profile
        const experimentId: string = getExperimentId();
        this.experimentProfile = await this.dataStore.getExperimentProfile(experimentId);
SparkSnail's avatar
SparkSnail committed
209
210
211
212
        this.readonly = readonly;
        if (readonly) {
            return Promise.resolve();
        }
Deshui Yu's avatar
Deshui Yu committed
213
        const expParams: ExperimentParams = this.experimentProfile.params;
214

215
        // Set up multiphase config
216
        if (expParams.multiPhase && this.trainingService.isMultiPhaseJobSupported) {
217
218
219
            this.trainingService.setClusterMetadata('multiPhase', expParams.multiPhase.toString());
        }

220
221
        // Set up versionCheck config
        if (expParams.versionCheck !== undefined) {
SparkSnail's avatar
SparkSnail committed
222
            this.trainingService.setClusterMetadata('version_check', expParams.versionCheck.toString());
223
224
        }

chicm-ms's avatar
chicm-ms committed
225
        const dispatcherCommand: string = getMsgDispatcherCommand(expParams);
226
        this.log.debug(`dispatcher command: ${dispatcherCommand}`);
QuanluZhang's avatar
QuanluZhang committed
227
        const checkpointDir: string = await this.createCheckpointDir();
Deshui Yu's avatar
Deshui Yu committed
228
        this.setupTuner(
229
230
            dispatcherCommand,
            undefined,
Deshui Yu's avatar
Deshui Yu committed
231
            'resume',
QuanluZhang's avatar
QuanluZhang committed
232
            checkpointDir);
Deshui Yu's avatar
Deshui Yu committed
233
234
235
236
237
238
239
240
241

        const allTrialJobs: TrialJobInfo[] = await this.dataStore.listTrialJobs();

        // Resume currSubmittedTrialNum
        this.currSubmittedTrialNum = allTrialJobs.length;

        // Check the final status for WAITING and RUNNING jobs
        await Promise.all(allTrialJobs
            .filter((job: TrialJobInfo) => job.status === 'WAITING' || job.status === 'RUNNING')
J-shang's avatar
J-shang committed
242
            .map((job: TrialJobInfo) => this.dataStore.storeTrialJobEvent('FAILED', job.trialJobId)));
Deshui Yu's avatar
Deshui Yu committed
243

244
245
246
        // Collect generated trials and imported trials
        const finishedTrialData: string = await this.exportData();
        const importedData: string[] = await this.dataStore.getImportedData();
chicm-ms's avatar
chicm-ms committed
247
        let trialData: Record<string, any>[] = JSON.parse(finishedTrialData);
248
249
        for (const oneImportedData of importedData) {
            // do not deduplicate
chicm-ms's avatar
chicm-ms committed
250
            trialData = trialData.concat(<Record<string, any>[]>JSON.parse(oneImportedData));
251
252
253
        }
        this.trialDataForTuner = JSON.stringify(trialData);

chicm-ms's avatar
chicm-ms committed
254
255
256
257
258
        if (this.experimentProfile.execDuration < this.experimentProfile.params.maxExecDuration &&
            this.currSubmittedTrialNum < this.experimentProfile.params.maxTrialNum &&
            this.experimentProfile.endTime) {
            delete this.experimentProfile.endTime;
        }
chicm-ms's avatar
chicm-ms committed
259
        this.setStatus('RUNNING');
260

Deshui Yu's avatar
Deshui Yu committed
261
        // TO DO: update database record for resume event
262
263
264
        this.run().catch((err: Error) => {
            this.criticalError(err);
        });
Deshui Yu's avatar
Deshui Yu committed
265
266
    }

267
268
    public getTrialJob(trialJobId: string): Promise<TrialJobInfo> {
        return this.dataStore.getTrialJob(trialJobId);
Deshui Yu's avatar
Deshui Yu committed
269
270
271
    }

    public async setClusterMetadata(key: string, value: string): Promise<void> {
SparkSnail's avatar
SparkSnail committed
272
273
274
        if (this.readonly) {
            return Promise.reject(new Error('Error: can not set cluster metadata in readonly mode!'));
        }
chicm-ms's avatar
chicm-ms committed
275
        this.log.info(`NNIManager setClusterMetadata, key: ${key}, value: ${value}`);
Deshui Yu's avatar
Deshui Yu committed
276
277
278
279
        let timeoutId: NodeJS.Timer;
        // TO DO: move timeout value to constants file
        const delay1: Promise<{}> = new Promise((resolve: Function, reject: Function): void => {
            timeoutId = setTimeout(
280
                () => { reject(new Error('TrainingService setClusterMetadata timeout. Please check your config file.')); },
281
                30000);
Deshui Yu's avatar
Deshui Yu committed
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
        });
        await Promise.race([delay1, this.trainingService.setClusterMetadata(key, value)]).finally(() => {
            clearTimeout(timeoutId);
        });
    }

    public getClusterMetadata(key: string): Promise<string> {
        return Promise.resolve(
            this.trainingService.getClusterMetadata(key)
        );
    }

    public async getTrialJobStatistics(): Promise<TrialJobStatistics[]> {
        return this.dataStore.getTrialJobStatistics();
    }

298
    public async stopExperiment(): Promise<void> {
chicm-ms's avatar
chicm-ms committed
299
300
        this.setStatus('STOPPING');
        this.log.info('Stopping experiment, cleaning up ...');
301
        await this.experimentDoneCleanUp();
chicm-ms's avatar
chicm-ms committed
302
        this.log.info('Experiment stopped.');
Deshui Yu's avatar
Deshui Yu committed
303
304
    }

305
    public async getMetricData(trialJobId?: string, metricType?: MetricType): Promise<MetricDataRecord[]> {
Deshui Yu's avatar
Deshui Yu committed
306
307
308
        return this.dataStore.getMetricData(trialJobId, metricType);
    }

309
310
311
312
313
314
    public async getMetricDataByRange(minSeqId: number, maxSeqId: number): Promise<MetricDataRecord[]> {
        const trialJobs = await this.dataStore.listTrialJobs();
        const targetTrials = trialJobs.filter(trial => (
            // FIXME: can this be undefined?
            trial.sequenceId !== undefined && minSeqId <= trial.sequenceId && trial.sequenceId <= maxSeqId
        ));
J-shang's avatar
J-shang committed
315
        const targetTrialIds = new Set(targetTrials.map(trial => trial.trialJobId));
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339

        const allMetrics = await this.dataStore.getMetricData();
        return allMetrics.filter(metric => targetTrialIds.has(metric.trialJobId));
    }

    public async getLatestMetricData(): Promise<MetricDataRecord[]> {
        // FIXME: this can take a long time
        const allMetrics: MetricDataRecord[] = await this.dataStore.getMetricData();
        const finals: MetricDataRecord[] = [];
        const latestIntermediates: Map<string, MetricDataRecord> = new Map<string, MetricDataRecord>();
        for (const metric of allMetrics) {
            if (metric.type !== 'PERIODICAL') {
                finals.push(metric);
            } else {
                const old: MetricDataRecord | undefined = latestIntermediates.get(metric.trialJobId);
                if (old === undefined || old.sequence <= metric.sequence) {
                    latestIntermediates.set(metric.trialJobId, metric);
                }
            }
        }
        return finals.concat(Array.from(latestIntermediates.values()));
        // FIXME: unit test
    }

340
341
342
343
    public async getTrialLog(trialJobId: string, logType: LogType): Promise<string> {
        return this.trainingService.getTrialLog(trialJobId, logType);
    }

Deshui Yu's avatar
Deshui Yu committed
344
345
346
347
348
349
350
351
    public getExperimentProfile(): Promise<ExperimentProfile> {
        // TO DO: using Promise.resolve()
        const deferred: Deferred<ExperimentProfile> = new Deferred<ExperimentProfile>();
        deferred.resolve(this.experimentProfile);

        return deferred.promise;
    }

352
353
354
355
    public getStatus(): NNIManagerStatus {
        return this.status;
    }

356
357
358
359
360
361
362
363
    public getTrialJobMessage(trialJobId: string): string | undefined {
        const trialJob = this.trialJobs.get(trialJobId);
        if (trialJob !== undefined){
            return trialJob.message
        }
        return undefined
    }

Deshui Yu's avatar
Deshui Yu committed
364
365
366
367
    public async listTrialJobs(status?: TrialJobStatus): Promise<TrialJobInfo[]> {
        return this.dataStore.listTrialJobs(status);
    }

368
369
    private setupTuner(command: string, cwd: string | undefined, mode: 'start' | 'resume', dataDirectory: string): void {
        if (this.dispatcher !== undefined) {
Deshui Yu's avatar
Deshui Yu committed
370
371
            return;
        }
goooxu's avatar
goooxu committed
372
        const stdio: StdioOptions = ['ignore', process.stdout, process.stderr, 'pipe', 'pipe'];
Deshui Yu's avatar
Deshui Yu committed
373
374
375
376
377
378
379
        let newCwd: string;
        if (cwd === undefined || cwd === '') {
            newCwd = getLogDir();
        } else {
            newCwd = cwd;
        }
        // TO DO: add CUDA_VISIBLE_DEVICES
380
381
382
383
384
        let includeIntermediateResultsEnv: boolean | undefined = false;
        if (this.experimentProfile.params.tuner !== undefined) {
            includeIntermediateResultsEnv = this.experimentProfile.params.tuner.includeIntermediateResults;
        }

chicm-ms's avatar
chicm-ms committed
385
        const nniEnv = {
chicm-ms's avatar
chicm-ms committed
386
            SDK_PROCESS: 'dispatcher',
Zejun Lin's avatar
Zejun Lin committed
387
388
            NNI_MODE: mode,
            NNI_CHECKPOINT_DIRECTORY: dataDirectory,
389
            NNI_LOG_DIRECTORY: getLogDir(),
390
            NNI_LOG_LEVEL: getLogLevel(),
391
392
            NNI_INCLUDE_INTERMEDIATE_RESULTS: includeIntermediateResultsEnv,
            CUDA_VISIBLE_DEVICES: this.getGpuEnvvarValue()
Zejun Lin's avatar
Zejun Lin committed
393
        };
chicm-ms's avatar
chicm-ms committed
394
        const newEnv = Object.assign({}, process.env, nniEnv);
395
        const tunerProc: ChildProcess = getTunerProc(command, stdio, newCwd, newEnv);
396
397
        this.dispatcherPid = tunerProc.pid;
        this.dispatcher = createDispatcherInterface(tunerProc);
Deshui Yu's avatar
Deshui Yu committed
398
399
400
401

        return;
    }

402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
    private getGpuEnvvarValue(): string {
        let cudaDevices: string | undefined;

        if (this.experimentProfile.params.advisor !== undefined) {
            cudaDevices = this.experimentProfile.params.advisor.gpuIndices;
        } else if (this.experimentProfile.params.tuner !== undefined) {
            cudaDevices = this.experimentProfile.params.tuner.gpuIndices;
        }

        if (cudaDevices === undefined) {
            return '';
        } else {
            return cudaDevices;
        }
    }

Deshui Yu's avatar
Deshui Yu committed
418
    private updateTrialConcurrency(trialConcurrency: number): void {
QuanluZhang's avatar
QuanluZhang committed
419
420
        // we assume trialConcurrency >= 0, which is checked by restserver
        this.trialConcurrencyChange += (trialConcurrency - this.experimentProfile.params.trialConcurrency);
Deshui Yu's avatar
Deshui Yu committed
421
422
423
424
425
426
427
428
429
430
431
432
        this.experimentProfile.params.trialConcurrency = trialConcurrency;

        return;
    }

    private updateMaxExecDuration(duration: number): void {
        this.experimentProfile.params.maxExecDuration = duration;

        return;
    }

    private updateSearchSpace(searchSpace: string): void {
433
        if (this.dispatcher === undefined) {
Deshui Yu's avatar
Deshui Yu committed
434
435
            throw new Error('Error: tuner has not been setup');
        }
436
        this.dispatcher.sendCommand(UPDATE_SEARCH_SPACE, searchSpace);
Deshui Yu's avatar
Deshui Yu committed
437
438
439
440
441
        this.experimentProfile.params.searchSpace = searchSpace;

        return;
    }

QuanluZhang's avatar
QuanluZhang committed
442
443
444
445
446
447
    private updateMaxTrialNum(maxTrialNum: number): void {
        this.experimentProfile.params.maxTrialNum = maxTrialNum;

        return;
    }

Deshui Yu's avatar
Deshui Yu committed
448
    private async experimentDoneCleanUp(): Promise<void> {
449
        if (this.dispatcher === undefined) {
Deshui Yu's avatar
Deshui Yu committed
450
451
            throw new Error('Error: tuner has not been setup');
        }
452
        this.trainingService.removeTrialJobMetricListener(this.trialJobMetricListener);
453
        this.dispatcher.sendCommand(TERMINATE);
Deshui Yu's avatar
Deshui Yu committed
454
455
456
        let tunerAlive: boolean = true;
        // gracefully terminate tuner and assessor here, wait at most 30 seconds.
        for (let i: number = 0; i < 30; i++) {
457
            if (!tunerAlive) { break; }
458
            tunerAlive = await isAlive(this.dispatcherPid);
Deshui Yu's avatar
Deshui Yu committed
459
460
            await delay(1000);
        }
461
        await killPid(this.dispatcherPid);
Deshui Yu's avatar
Deshui Yu committed
462
        const trialJobList: TrialJobDetail[] = await this.trainingService.listTrialJobs();
463
464
465

        // DON'T try to make it in parallel, the training service may not handle it well.
        // If there is performance concern, consider to support batch cancellation on training service.
Deshui Yu's avatar
Deshui Yu committed
466
467
468
469
        for (const trialJob of trialJobList) {
            if (trialJob.status === 'RUNNING' ||
                trialJob.status === 'WAITING') {
                try {
chicm-ms's avatar
chicm-ms committed
470
                    this.log.info(`cancelTrialJob: ${trialJob.id}`);
Deshui Yu's avatar
Deshui Yu committed
471
472
                    await this.trainingService.cancelTrialJob(trialJob.id);
                } catch (error) {
473
                    this.log.debug(`ignorable error on canceling trial ${trialJob.id}. ${error}`);
Deshui Yu's avatar
Deshui Yu committed
474
475
476
477
                }
            }
        }
        await this.trainingService.cleanUp();
478
479
480
        if (this.experimentProfile.endTime === undefined) {
            this.setEndtime();
        }
Deshui Yu's avatar
Deshui Yu committed
481
        await this.storeExperimentProfile();
chicm-ms's avatar
chicm-ms committed
482
        this.setStatus('STOPPED');
J-shang's avatar
J-shang committed
483
        this.experimentManager.setExperimentInfo(this.experimentProfile.id, 'port', undefined);
Deshui Yu's avatar
Deshui Yu committed
484
485
486
    }

    private async periodicallyUpdateExecDuration(): Promise<void> {
487
        let count: number = 1;
488
        while (!['ERROR', 'STOPPING', 'STOPPED'].includes(this.status.status)) {
489
            await delay(1000 * 1); // 1 seconds
490
            if (['RUNNING', 'NO_MORE_TRIAL', 'TUNER_NO_MORE_TRIAL'].includes(this.status.status)) {
491
492
493
494
495
496
                this.experimentProfile.execDuration += 1;
                if (count % 10 === 0) {
                    await this.storeExperimentProfile();
                }
            }
            count += 1;
Deshui Yu's avatar
Deshui Yu committed
497
498
499
        }
    }

chicm-ms's avatar
chicm-ms committed
500
501
502
503
504
505
    private async pingDispatcher(): Promise<void> {
        if (this.dispatcher === undefined) {
            throw new Error('Error: tuner has not been setup');
        }
        while (!['ERROR', 'STOPPING', 'STOPPED'].includes(this.status.status)) {
            this.dispatcher.sendCommand(PING);
chicm-ms's avatar
chicm-ms committed
506
            await delay(1000 * 5);
chicm-ms's avatar
chicm-ms committed
507
508
509
        }
    }

QuanluZhang's avatar
QuanluZhang committed
510
511
    private async requestTrialJobsStatus(): Promise<number> {
        let finishedTrialJobNum: number = 0;
QuanluZhang's avatar
QuanluZhang committed
512
513
514
        if (this.dispatcher === undefined) {
            throw new Error('Error: tuner has not been setup');
        }
QuanluZhang's avatar
QuanluZhang committed
515
516
517
518
        for (const trialJobId of Array.from(this.trialJobs.keys())) {
            const trialJobDetail: TrialJobDetail = await this.trainingService.getTrialJob(trialJobId);
            const oldTrialJobDetail: TrialJobDetail | undefined = this.trialJobs.get(trialJobId);
            if (oldTrialJobDetail !== undefined && oldTrialJobDetail.status !== trialJobDetail.status) {
chicm-ms's avatar
chicm-ms committed
519
                this.log.info(`Trial job ${trialJobDetail.id} status changed from ${oldTrialJobDetail.status} to ${trialJobDetail.status}`);
QuanluZhang's avatar
QuanluZhang committed
520
                this.trialJobs.set(trialJobId, Object.assign({}, trialJobDetail));
521
                await this.dataStore.storeTrialJobEvent(trialJobDetail.status, trialJobDetail.id, undefined, trialJobDetail);
QuanluZhang's avatar
QuanluZhang committed
522
            }
523
524
525
526
            const newTrialJobDetail: TrialJobDetail | undefined = this.trialJobs.get(trialJobId);
            if (newTrialJobDetail !== undefined) {
                newTrialJobDetail.message = trialJobDetail.message;
            }
QuanluZhang's avatar
QuanluZhang committed
527
            let hyperParams: string | undefined = undefined;
QuanluZhang's avatar
QuanluZhang committed
528
529
530
            switch (trialJobDetail.status) {
                case 'SUCCEEDED':
                case 'USER_CANCELED':
QuanluZhang's avatar
QuanluZhang committed
531
                case 'EARLY_STOPPED':
QuanluZhang's avatar
QuanluZhang committed
532
533
                    this.trialJobs.delete(trialJobId);
                    finishedTrialJobNum++;
534
                    hyperParams = trialJobDetail.form.hyperParameters.value;
QuanluZhang's avatar
QuanluZhang committed
535
                    this.dispatcher.sendCommand(TRIAL_END, JSON.stringify({
chicm-ms's avatar
chicm-ms committed
536
                        trial_job_id: trialJobDetail.id, // eslint-disable-line @typescript-eslint/camelcase
QuanluZhang's avatar
QuanluZhang committed
537
                        event: trialJobDetail.status,
chicm-ms's avatar
chicm-ms committed
538
                        hyper_params: hyperParams // eslint-disable-line @typescript-eslint/camelcase
goooxu's avatar
goooxu committed
539
                    }));
QuanluZhang's avatar
QuanluZhang committed
540
541
542
543
544
545
546
                    break;
                case 'FAILED':
                case 'SYS_CANCELED':
                    // In the current version, we do not retry
                    // TO DO: push this job to queue for retry
                    this.trialJobs.delete(trialJobId);
                    finishedTrialJobNum++;
547
                    hyperParams = trialJobDetail.form.hyperParameters.value;
QuanluZhang's avatar
QuanluZhang committed
548
                    this.dispatcher.sendCommand(TRIAL_END, JSON.stringify({
chicm-ms's avatar
chicm-ms committed
549
                        trial_job_id: trialJobDetail.id, // eslint-disable-line @typescript-eslint/camelcase
QuanluZhang's avatar
QuanluZhang committed
550
                        event: trialJobDetail.status,
chicm-ms's avatar
chicm-ms committed
551
                        hyper_params: hyperParams // eslint-disable-line @typescript-eslint/camelcase
goooxu's avatar
goooxu committed
552
                    }));
QuanluZhang's avatar
QuanluZhang committed
553
554
555
556
557
558
559
560
561
562
                    break;
                case 'WAITING':
                case 'RUNNING':
                case 'UNKNOWN':
                    // Do nothing
                    break;
                default:
                // TO DO: add warning in log
            }
        }
goooxu's avatar
goooxu committed
563

Gems Guo's avatar
Gems Guo committed
564
        return finishedTrialJobNum;
QuanluZhang's avatar
QuanluZhang committed
565
566
567
568
569
570
    }

    private async manageTrials(): Promise<void> {
        if (this.dispatcher === undefined) {
            throw new Error('Error: tuner has not been setup');
        }
QuanluZhang's avatar
QuanluZhang committed
571
        let allFinishedTrialJobNum: number = this.currSubmittedTrialNum;
QuanluZhang's avatar
QuanluZhang committed
572
        let waitSubmittedToFinish: number;
573
        while (!['ERROR', 'STOPPING', 'STOPPED'].includes(this.status.status)) {
QuanluZhang's avatar
QuanluZhang committed
574
575
576
577
578
579
580
            const finishedTrialJobNum: number = await this.requestTrialJobsStatus();
            allFinishedTrialJobNum += finishedTrialJobNum;

            // requestTrialNum is the number of trials that will be requested from tuner.
            // If trialConcurrency does not change, requestTrialNum equals finishedTrialJobNum.
            // If trialConcurrency changes, for example, trialConcurrency increases by 2 (trialConcurrencyChange=2), then
            // requestTrialNum equals 2 + finishedTrialJobNum and trialConcurrencyChange becomes 0.
581
            // If trialConcurrency changes, for example, trialConcurrency decreases by 4 (trialConcurrencyChange=-4) and
QuanluZhang's avatar
QuanluZhang committed
582
583
584
585
586
587
588
589
            // finishedTrialJobNum is 2, then requestTrialNum becomes -2. No trial will be requested from tuner,
            // and trialConcurrencyChange becomes -2.
            const requestTrialNum: number = this.trialConcurrencyChange + finishedTrialJobNum;
            if (requestTrialNum >= 0) {
                this.trialConcurrencyChange = 0;
            } else {
                this.trialConcurrencyChange = requestTrialNum;
            }
chicm-ms's avatar
chicm-ms committed
590

591
            this.requestTrialJobs(requestTrialNum);
chicm-ms's avatar
chicm-ms committed
592

QuanluZhang's avatar
QuanluZhang committed
593
            // check maxtrialnum and maxduration here
594
            // NO_MORE_TRIAL is more like a subset of RUNNING, because during RUNNING tuner
595
            // might tell nnimanager that this is no more trials. In NO_MORE_TRIAL state, the experiment is viewed
596
597
            // as still running. DONE could be transfered from RUNNING or NO_MORE_TRIAL.
            assert(this.status.status === 'RUNNING' ||
598
                this.status.status === 'DONE' ||
QuanluZhang's avatar
QuanluZhang committed
599
                this.status.status === 'NO_MORE_TRIAL' ||
600
                this.status.status === 'TUNER_NO_MORE_TRIAL', `Actual status: ${this.status.status}`);
601
            if (this.experimentProfile.execDuration > this.experimentProfile.params.maxExecDuration ||
QuanluZhang's avatar
QuanluZhang committed
602
                this.currSubmittedTrialNum >= this.experimentProfile.params.maxTrialNum) {
QuanluZhang's avatar
QuanluZhang committed
603
                if (this.status.status !== 'DONE') {
chicm-ms's avatar
chicm-ms committed
604
                    this.setStatus('NO_MORE_TRIAL');
QuanluZhang's avatar
QuanluZhang committed
605
606
607
608
                    waitSubmittedToFinish = this.currSubmittedTrialNum;

                    assert(allFinishedTrialJobNum <= waitSubmittedToFinish);
                    if (allFinishedTrialJobNum >= waitSubmittedToFinish) {
chicm-ms's avatar
chicm-ms committed
609
                        this.setStatus('DONE');
610
                        this.setEndtime();
QuanluZhang's avatar
QuanluZhang committed
611
612
613
614
                        await this.storeExperimentProfile();
                        // write this log for travis CI
                        this.log.info('Experiment done.');
                    }
QuanluZhang's avatar
QuanluZhang committed
615
616
617
                }
            } else {
                if (this.status.status === 'DONE') {
618
619
                    delete this.experimentProfile.endTime;
                    await this.storeExperimentProfile();
QuanluZhang's avatar
QuanluZhang committed
620
                }
QuanluZhang's avatar
QuanluZhang committed
621
                if (this.status.status !== 'TUNER_NO_MORE_TRIAL') {
chicm-ms's avatar
chicm-ms committed
622
                    this.setStatus('RUNNING');
623
                }
QuanluZhang's avatar
QuanluZhang committed
624
625
626
627
628
                for (let i: number = this.trialJobs.size; i < this.experimentProfile.params.trialConcurrency; i++) {
                    if (this.waitingTrials.length === 0 ||
                        this.currSubmittedTrialNum >= this.experimentProfile.params.maxTrialNum) {
                        break;
                    }
629
                    const form = this.waitingTrials.shift() as TrialJobApplicationForm;
QuanluZhang's avatar
QuanluZhang committed
630
                    this.currSubmittedTrialNum++;
631
632
                    this.log.info(`submitTrialJob: form: ${JSON.stringify(form)}`);
                    const trialJobDetail: TrialJobDetail = await this.trainingService.submitTrialJob(form);
633
                    await this.storeExperimentProfile();
QuanluZhang's avatar
QuanluZhang committed
634
635
636
637
                    this.trialJobs.set(trialJobDetail.id, Object.assign({}, trialJobDetail));
                    const trialJobDetailSnapshot: TrialJobDetail | undefined = this.trialJobs.get(trialJobDetail.id);
                    if (trialJobDetailSnapshot != undefined) {
                        await this.dataStore.storeTrialJobEvent(
638
                            trialJobDetailSnapshot.status, trialJobDetailSnapshot.id, form.hyperParameters.value, trialJobDetailSnapshot);
QuanluZhang's avatar
QuanluZhang committed
639
640
641
642
643
644
645
646
647
                    } else {
                        assert(false, `undefined trialJobDetail in trialJobs: ${trialJobDetail.id}`);
                    }
                }
            }
            await delay(1000 * 5); // 5 seconds
        }
    }

Deshui Yu's avatar
Deshui Yu committed
648
649
650
651
652
653
    private storeExperimentProfile(): Promise<void> {
        this.experimentProfile.revision += 1;

        return this.dataStore.storeExperimentProfile(this.experimentProfile);
    }

654
    private async run(): Promise<void> {
QuanluZhang's avatar
QuanluZhang committed
655
        assert(this.dispatcher !== undefined);
656
657
658
659
660
661
662

        this.addEventListeners();

        this.sendInitTunerCommands();

        await Promise.all([
            this.periodicallyUpdateExecDuration(),
chicm-ms's avatar
chicm-ms committed
663
            this.pingDispatcher().catch((err: Error) => {
chicm-ms's avatar
chicm-ms committed
664
                throw NNIError.FromError(err, 'Dispatcher error: ');
chicm-ms's avatar
chicm-ms committed
665
            }),
666
            this.trainingService.run().catch((err: Error) => {
chicm-ms's avatar
chicm-ms committed
667
                throw NNIError.FromError(err, 'Training service error: ');
668
            }),
QuanluZhang's avatar
QuanluZhang committed
669
            this.manageTrials().catch((err: Error) => {
chicm-ms's avatar
chicm-ms committed
670
                throw NNIError.FromError(err, 'Job management error: ');
671
            })]);
672
673
    }

QuanluZhang's avatar
QuanluZhang committed
674
    private addEventListeners(): void {
chicm-ms's avatar
chicm-ms committed
675
        this.log.info('Add event listeners');
676
        // TO DO: cannot run this method more than once in one NNIManager instance
QuanluZhang's avatar
QuanluZhang committed
677
        if (this.dispatcher === undefined) {
678
679
            throw new Error('Error: tuner or job maintainer have not been setup');
        }
680
        this.trainingService.addTrialJobMetricListener(this.trialJobMetricListener);
681
682
683

        this.dispatcher.onCommand((commandType: string, content: string) => {
            this.onTunerCommand(commandType, content).catch((err: Error) => {
chicm-ms's avatar
chicm-ms committed
684
                this.criticalError(NNIError.FromError(err, 'Tuner command event error: '));
685
686
            });
        });
687
688
689
690
        this.dispatcher.onError((error: Error) => {
            this.log.error(`Dispatcher error: ${error.message}`);
            this.criticalError(new Error('Dispatcher stream error, tuner may have crashed.'));
        });
691
692
693
694
    }

    private sendInitTunerCommands(): void {
        if (this.dispatcher === undefined) {
695
            throw new Error('Dispatcher error: tuner has not been setup');
696
        }
chicm-ms's avatar
chicm-ms committed
697
698
699
        this.log.debug(`Send tuner command: INITIALIZE: ${this.experimentProfile.params.searchSpace}`);
        // Tuner need to be initialized with search space before generating any hyper parameters
        this.dispatcher.sendCommand(INITIALIZE, this.experimentProfile.params.searchSpace);
700
701
702
    }

    private async onTrialJobMetrics(metric: TrialJobMetric): Promise<void> {
703
        this.log.debug(`NNIManager received trial job metrics: ${JSON.stringify(metric)}`);
704
705
706
707
708
709
710
711
        if (this.trialJobs.has(metric.id)){
            await this.dataStore.storeMetricData(metric.id, metric.data);
            if (this.dispatcher === undefined) {
                throw new Error('Error: tuner has not been setup');
            }
            this.dispatcher.sendCommand(REPORT_METRIC_DATA, metric.data);
        } else {
            this.log.warning(`NNIManager received non-existent trial job metrics: ${metric}`);
712
713
714
        }
    }

chicm-ms's avatar
chicm-ms committed
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
    private requestTrialJobs(jobNum: number): void {
        if (jobNum < 1) {
            return;
        }
        if (this.dispatcher === undefined) {
            throw new Error('Dispatcher error: tuner has not been setup');
        }
        if (this.experimentProfile.params.multiThread) {
            // Send multiple requests to ensure multiple hyper parameters are generated in non-blocking way.
            // For a single REQUEST_TRIAL_JOBS request, hyper parameters are generated one by one
            // sequentially.
            for (let i: number = 0; i < jobNum; i++) {
                this.dispatcher.sendCommand(REQUEST_TRIAL_JOBS, '1');
            }
        } else {
            this.dispatcher.sendCommand(REQUEST_TRIAL_JOBS, String(jobNum));
        }
    }

734
    private async onTunerCommand(commandType: string, content: string): Promise<void> {
horizon365's avatar
horizon365 committed
735
        this.log.info(`NNIManager received command from dispatcher: ${commandType}, ${content}`);
736
        switch (commandType) {
chicm-ms's avatar
chicm-ms committed
737
            case INITIALIZED: {
chicm-ms's avatar
chicm-ms committed
738
                // Tuner is intialized, search space is set, request tuner to generate hyper parameters
739
740
741
742
743
744
                if (this.trialDataForTuner.length > 0) {
                    if (this.dispatcher === undefined) {
                        throw new Error('Dispatcher error: tuner has not been setup');
                    }
                    this.dispatcher.sendCommand(IMPORT_DATA, this.trialDataForTuner);
                }
chicm-ms's avatar
chicm-ms committed
745
746
                this.requestTrialJobs(this.experimentProfile.params.trialConcurrency);
                break;
chicm-ms's avatar
chicm-ms committed
747
748
            }
            case NEW_TRIAL_JOB: {
QuanluZhang's avatar
QuanluZhang committed
749
                if (this.status.status === 'TUNER_NO_MORE_TRIAL') {
750
                    this.log.warning('It is not supposed to receive more trials after NO_MORE_TRIAL is set');
chicm-ms's avatar
chicm-ms committed
751
                    this.setStatus('RUNNING');
752
                }
753
754
755
756
757
758
759
760
                const form: TrialJobApplicationForm = {
                    sequenceId: this.experimentProfile.nextSequenceId++,
                    hyperParameters: {
                        value: content,
                        index: 0
                    }
                };
                this.waitingTrials.push(form);
761
                break;
chicm-ms's avatar
chicm-ms committed
762
763
            }
            case SEND_TRIAL_JOB_PARAMETER: {
chicm-ms's avatar
chicm-ms committed
764
765
766
767
768
                const tunerCommand: any = JSON.parse(content);
                assert(tunerCommand.parameter_index >= 0);
                assert(tunerCommand.trial_job_id !== undefined);

                const trialJobForm: TrialJobApplicationForm = {
769
                    sequenceId: -1,  // FIXME: multi-phase tuner should use sequence ID instead of trial job ID
chicm-ms's avatar
chicm-ms committed
770
771
772
773
774
                    hyperParameters: {
                        value: content,
                        index: tunerCommand.parameter_index
                    }
                };
chicm-ms's avatar
chicm-ms committed
775
                this.log.info(`updateTrialJob: job id: ${tunerCommand.trial_job_id}, form: ${JSON.stringify(trialJobForm)}`);
chicm-ms's avatar
chicm-ms committed
776
                await this.trainingService.updateTrialJob(tunerCommand.trial_job_id, trialJobForm);
777
778
779
780
781
                if (tunerCommand['parameters'] !== null) {
                    // parameters field is set as empty string if no more hyper parameter can be generated by tuner.
                    await this.dataStore.storeTrialJobEvent(
                        'ADD_HYPERPARAMETER', tunerCommand.trial_job_id, content, undefined);
                }
chicm-ms's avatar
chicm-ms committed
782
                break;
chicm-ms's avatar
chicm-ms committed
783
784
            }
            case NO_MORE_TRIAL_JOBS: {
785
786
787
                if (!['ERROR', 'STOPPING', 'STOPPED'].includes(this.status.status)) {
                    this.setStatus('TUNER_NO_MORE_TRIAL');
                }
788
                break;
chicm-ms's avatar
chicm-ms committed
789
790
            }
            case KILL_TRIAL_JOB: {
chicm-ms's avatar
chicm-ms committed
791
                this.log.info(`cancelTrialJob: ${JSON.parse(content)}`);
QuanluZhang's avatar
QuanluZhang committed
792
                await this.trainingService.cancelTrialJob(JSON.parse(content), true);
793
                break;
chicm-ms's avatar
chicm-ms committed
794
            }
795
796
797
            default:
                throw new Error('Error: unsupported command type from tuner');
        }
Deshui Yu's avatar
Deshui Yu committed
798
799
    }

800
801
802
803
804
805
806
807
808
809
    private criticalError(err: Error): void {
        this.logError(err);
        console.error(err);
    }

    private logError(err: Error): void {
        if (err.stack !== undefined) {
            this.log.error(err.stack);
        }
        this.status.errors.push(err.message);
810
        this.setEndtime();
chicm-ms's avatar
chicm-ms committed
811
812
813
814
815
816
817
        this.setStatus('ERROR');
    }

    private setStatus(status: ExperimentStatus): void {
        if (status !== this.status.status) {
            this.log.info(`Change NNIManager status from: ${this.status.status} to: ${status}`);
            this.status.status = status;
818
            this.experimentManager.setExperimentInfo(this.experimentProfile.id, 'status', this.status.status);
chicm-ms's avatar
chicm-ms committed
819
        }
820
821
    }

822
823
824
825
826
    private setEndtime(): void {
        this.experimentProfile.endTime = Date.now();
        this.experimentManager.setExperimentInfo(this.experimentProfile.id, 'endTime', this.experimentProfile.endTime);
    }

827
828
829
830
831
    private createEmptyExperimentProfile(): ExperimentProfile {
        return {
            id: getExperimentId(),
            revision: 0,
            execDuration: 0,
832
            logDir: getExperimentRootDir(),
833
            nextSequenceId: 0,
834
835
836
837
838
839
            params: {
                authorName: '',
                experimentName: '',
                trialConcurrency: 0,
                maxExecDuration: 0, // unit: second
                maxTrialNum: 0, // maxTrialNum includes all the submitted trial jobs
840
                trainingServicePlatform: '',
QuanluZhang's avatar
QuanluZhang committed
841
                searchSpace: ''
842
843
            }
        };
Deshui Yu's avatar
Deshui Yu committed
844
    }
845

QuanluZhang's avatar
QuanluZhang committed
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
    private async createCheckpointDir(): Promise<string> {
        // TODO: test
        const chkpDir: string = getCheckpointDir();
        // create checkpoint directory
        await mkDirP(chkpDir);
        // assign this directory to exp profile's checkpointDir
        if (this.experimentProfile.params.advisor) {
            this.experimentProfile.params.advisor.checkpointDir = chkpDir;
        }
        if (this.experimentProfile.params.tuner) {
            this.experimentProfile.params.tuner.checkpointDir = chkpDir;
        }
        if (this.experimentProfile.params.assessor) {
            this.experimentProfile.params.assessor.checkpointDir = chkpDir;
        }

        return Promise.resolve(chkpDir);
    }
Deshui Yu's avatar
Deshui Yu committed
864
865
866
}

export { NNIManager };