restHandler.ts 12.9 KB
Newer Older
liuzhe-lz's avatar
liuzhe-lz committed
1
2
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
Deshui Yu's avatar
Deshui Yu committed
3
4
5
6
7
8
9
10
11

'use strict';

import { Request, Response, Router } from 'express';
import * as path from 'path';

import * as component from '../common/component';
import { DataStore, MetricDataRecord, TrialJobInfo } from '../common/datastore';
import { NNIError, NNIErrorNames } from '../common/errors';
SparkSnail's avatar
SparkSnail committed
12
import { isNewExperiment, isReadonly } from '../common/experimentStartupInfo';
Deshui Yu's avatar
Deshui Yu committed
13
import { getLogger, Logger } from '../common/log';
chicm-ms's avatar
chicm-ms committed
14
import { ExperimentProfile, Manager, TrialJobStatistics } from '../common/manager';
15
import { ExperimentManager } from '../common/experimentManager';
16
17
import { ValidationSchemas } from './restValidationSchemas';
import { NNIRestServer } from './nniRestServer';
18
import { getVersion } from '../common/utils';
Deshui Yu's avatar
Deshui Yu committed
19

20
21
const expressJoi = require('express-joi-validator');

Deshui Yu's avatar
Deshui Yu committed
22
class NNIRestHandler {
23
    private restServer: NNIRestServer;
24
25
    private nniManager: Manager;
    private experimentsManager: ExperimentManager;
Deshui Yu's avatar
Deshui Yu committed
26
27
    private log: Logger;

28
    constructor(rs: NNIRestServer) {
Deshui Yu's avatar
Deshui Yu committed
29
        this.nniManager = component.get(Manager);
30
        this.experimentsManager = component.get(ExperimentManager);
Deshui Yu's avatar
Deshui Yu committed
31
32
33
34
35
36
37
38
        this.restServer = rs;
        this.log = getLogger();
    }

    public createRestHandler(): Router {
        const router: Router = Router();

        router.use((req: Request, res: Response, next) => {
chicm-ms's avatar
chicm-ms committed
39
            this.log.debug(`${req.method}: ${req.url}: body:\n${JSON.stringify(req.body, undefined, 4)}`);
Deshui Yu's avatar
Deshui Yu committed
40
41
42
43
44
45
46
47
            res.header('Access-Control-Allow-Origin', '*');
            res.header('Access-Control-Allow-Headers', 'Origin, X-Requested-With, Content-Type, Accept');
            res.header('Access-Control-Allow-Methods', 'PUT,POST,GET,DELETE,OPTIONS');

            res.setHeader('Content-Type', 'application/json');
            next();
        });

Gems Guo's avatar
Gems Guo committed
48
        this.version(router);
Deshui Yu's avatar
Deshui Yu committed
49
50
51
        this.checkStatus(router);
        this.getExperimentProfile(router);
        this.updateExperimentProfile(router);
52
        this.importData(router);
53
        this.getImportedData(router);
Deshui Yu's avatar
Deshui Yu committed
54
55
56
57
58
59
60
61
        this.startExperiment(router);
        this.getTrialJobStatistics(router);
        this.setClusterMetaData(router);
        this.listTrialJobs(router);
        this.getTrialJob(router);
        this.addTrialJob(router);
        this.cancelTrialJob(router);
        this.getMetricData(router);
62
63
        this.getMetricDataByRange(router);
        this.getLatestMetricData(router);
64
        this.getTrialLog(router);
65
        this.exportData(router);
66
        this.getExperimentsInfo(router);
Deshui Yu's avatar
Deshui Yu committed
67

68
        // Express-joi-validator configuration
69
        router.use((err: any, _req: Request, res: Response, _next: any) => {
70
71
72
73
74
75
76
            if (err.isBoom) {
                this.log.error(err.output.payload);

                return res.status(err.output.statusCode).json(err.output.payload);
            }
        });

Deshui Yu's avatar
Deshui Yu committed
77
78
79
        return router;
    }

chicm-ms's avatar
chicm-ms committed
80
    private handleError(err: Error, res: Response, isFatal: boolean = false, errorCode: number = 500): void {
Deshui Yu's avatar
Deshui Yu committed
81
82
83
        if (err instanceof NNIError && err.name === NNIErrorNames.NOT_FOUND) {
            res.status(404);
        } else {
SparkSnail's avatar
SparkSnail committed
84
            res.status(errorCode);
Deshui Yu's avatar
Deshui Yu committed
85
86
87
88
        }
        res.send({
            error: err.message
        });
89
90

        // If it's a fatal error, exit process
chicm-ms's avatar
chicm-ms committed
91
        if (isFatal) {
92
            this.log.fatal(err);
93
            process.exit(1);
chicm-ms's avatar
chicm-ms committed
94
95
        } else {
            this.log.error(err);
96
        }
Deshui Yu's avatar
Deshui Yu committed
97
98
    }

Gems Guo's avatar
Gems Guo committed
99
100
    private version(router: Router): void {
        router.get('/version', async (req: Request, res: Response) => {
101
102
            const version = await getVersion();
            res.send(version);
Gems Guo's avatar
Gems Guo committed
103
104
105
        });
    }

Deshui Yu's avatar
Deshui Yu committed
106
107
108
109
110
    // TODO add validators for request params, query, body
    private checkStatus(router: Router): void {
        router.get('/check-status', (req: Request, res: Response) => {
            const ds: DataStore = component.get<DataStore>(DataStore);
            ds.init().then(() => {
111
                res.send(this.nniManager.getStatus());
Deshui Yu's avatar
Deshui Yu committed
112
            }).catch(async (err: Error) => {
chicm-ms's avatar
chicm-ms committed
113
                this.handleError(err, res);
Deshui Yu's avatar
Deshui Yu committed
114
                this.log.error(err.message);
chicm-ms's avatar
chicm-ms committed
115
                this.log.error(`Datastore initialize failed, stopping rest server...`);
Deshui Yu's avatar
Deshui Yu committed
116
117
118
119
120
121
122
123
124
125
                await this.restServer.stop();
            });
        });
    }

    private getExperimentProfile(router: Router): void {
        router.get('/experiment', (req: Request, res: Response) => {
            this.nniManager.getExperimentProfile().then((profile: ExperimentProfile) => {
                res.send(profile);
            }).catch((err: Error) => {
chicm-ms's avatar
chicm-ms committed
126
                this.handleError(err, res);
Deshui Yu's avatar
Deshui Yu committed
127
128
129
130
131
            });
        });
    }

    private updateExperimentProfile(router: Router): void {
132
        router.put('/experiment', expressJoi(ValidationSchemas.UPDATEEXPERIMENT), (req: Request, res: Response) => {
Deshui Yu's avatar
Deshui Yu committed
133
134
135
            this.nniManager.updateExperimentProfile(req.body, req.query.update_type).then(() => {
                res.send();
            }).catch((err: Error) => {
chicm-ms's avatar
chicm-ms committed
136
                this.handleError(err, res);
Deshui Yu's avatar
Deshui Yu committed
137
138
139
            });
        });
    }
140

141
142
143
144
145
    private importData(router: Router): void {
        router.post('/experiment/import-data', (req: Request, res: Response) => {
            this.nniManager.importData(JSON.stringify(req.body)).then(() => {
                res.send();
            }).catch((err: Error) => {
chicm-ms's avatar
chicm-ms committed
146
                this.handleError(err, res);
147
148
149
            });
        });
    }
Deshui Yu's avatar
Deshui Yu committed
150

151
152
153
154
155
156
157
158
159
160
    private getImportedData(router: Router): void {
        router.get('/experiment/imported-data', (req: Request, res: Response) => {
            this.nniManager.getImportedData().then((importedData: string[]) => {
                res.send(JSON.stringify(importedData));
            }).catch((err: Error) => {
                this.handleError(err, res);
            });
        });
    }

Deshui Yu's avatar
Deshui Yu committed
161
    private startExperiment(router: Router): void {
162
        router.post('/experiment', expressJoi(ValidationSchemas.STARTEXPERIMENT), (req: Request, res: Response) => {
Deshui Yu's avatar
Deshui Yu committed
163
164
165
            if (isNewExperiment()) {
                this.nniManager.startExperiment(req.body).then((eid: string) => {
                    res.send({
chicm-ms's avatar
chicm-ms committed
166
                        experiment_id: eid // eslint-disable-line @typescript-eslint/camelcase
Deshui Yu's avatar
Deshui Yu committed
167
168
                    });
                }).catch((err: Error) => {
169
                    // Start experiment is a step of initialization, so any exception thrown is a fatal
chicm-ms's avatar
chicm-ms committed
170
                    this.handleError(err, res);
Deshui Yu's avatar
Deshui Yu committed
171
172
                });
            } else {
SparkSnail's avatar
SparkSnail committed
173
                this.nniManager.resumeExperiment(isReadonly()).then(() => {
Deshui Yu's avatar
Deshui Yu committed
174
175
                    res.send();
                }).catch((err: Error) => {
176
                    // Resume experiment is a step of initialization, so any exception thrown is a fatal
chicm-ms's avatar
chicm-ms committed
177
                    this.handleError(err, res);
Deshui Yu's avatar
Deshui Yu committed
178
                });
SparkSnail's avatar
SparkSnail committed
179
            } 
Deshui Yu's avatar
Deshui Yu committed
180
181
182
183
184
185
186
187
        });
    }

    private getTrialJobStatistics(router: Router): void {
        router.get('/job-statistics', (req: Request, res: Response) => {
            this.nniManager.getTrialJobStatistics().then((statistics: TrialJobStatistics[]) => {
                res.send(statistics);
            }).catch((err: Error) => {
chicm-ms's avatar
chicm-ms committed
188
                this.handleError(err, res);
Deshui Yu's avatar
Deshui Yu committed
189
190
191
192
193
            });
        });
    }

    private setClusterMetaData(router: Router): void {
194
195
196
        router.put(
            '/experiment/cluster-metadata', expressJoi(ValidationSchemas.SETCLUSTERMETADATA),
            async (req: Request, res: Response) => {
SparkSnail's avatar
SparkSnail committed
197
198
199
200
201
202
203
204
205
                const metadata: any = req.body;
                const keys: string[] = Object.keys(metadata);
                try {
                    for (const key of keys) {
                        await this.nniManager.setClusterMetadata(key, JSON.stringify(metadata[key]));
                    }
                    res.send();
                } catch (err) {
                    // setClusterMetata is a step of initialization, so any exception thrown is a fatal
chicm-ms's avatar
chicm-ms committed
206
                    this.handleError(NNIError.FromError(err), res, true);
Deshui Yu's avatar
Deshui Yu committed
207
208
209
210
211
212
213
214
215
                }
        });
    }

    private listTrialJobs(router: Router): void {
        router.get('/trial-jobs', (req: Request, res: Response) => {
            this.nniManager.listTrialJobs(req.query.status).then((jobInfos: TrialJobInfo[]) => {
                jobInfos.forEach((trialJob: TrialJobInfo) => {
                    this.setErrorPathForFailedJob(trialJob);
216
                    this.setMessageforJob(trialJob);
Deshui Yu's avatar
Deshui Yu committed
217
218
219
                });
                res.send(jobInfos);
            }).catch((err: Error) => {
chicm-ms's avatar
chicm-ms committed
220
                this.handleError(err, res);
Deshui Yu's avatar
Deshui Yu committed
221
222
223
224
225
226
227
228
            });
        });
    }

    private getTrialJob(router: Router): void {
        router.get('/trial-jobs/:id', (req: Request, res: Response) => {
            this.nniManager.getTrialJob(req.params.id).then((jobDetail: TrialJobInfo) => {
                const jobInfo: TrialJobInfo = this.setErrorPathForFailedJob(jobDetail);
229
                this.setMessageforJob(jobInfo);
Deshui Yu's avatar
Deshui Yu committed
230
231
                res.send(jobInfo);
            }).catch((err: Error) => {
chicm-ms's avatar
chicm-ms committed
232
                this.handleError(err, res);
Deshui Yu's avatar
Deshui Yu committed
233
234
235
236
237
238
            });
        });
    }

    private addTrialJob(router: Router): void {
        router.post('/trial-jobs', async (req: Request, res: Response) => {
239
240
            this.nniManager.addCustomizedTrialJob(JSON.stringify(req.body)).then((sequenceId: number) => {
                res.send({sequenceId});
Deshui Yu's avatar
Deshui Yu committed
241
            }).catch((err: Error) => {
chicm-ms's avatar
chicm-ms committed
242
                this.handleError(err, res);
Deshui Yu's avatar
Deshui Yu committed
243
244
245
246
247
248
249
250
251
            });
        });
    }

    private cancelTrialJob(router: Router): void {
        router.delete('/trial-jobs/:id', async (req: Request, res: Response) => {
            this.nniManager.cancelTrialJobByUser(req.params.id).then(() => {
                res.send();
            }).catch((err: Error) => {
chicm-ms's avatar
chicm-ms committed
252
                this.handleError(err, res);
Deshui Yu's avatar
Deshui Yu committed
253
254
255
256
257
            });
        });
    }

    private getMetricData(router: Router): void {
258
        router.get('/metric-data/:job_id*?', async (req: Request, res: Response) => {
Deshui Yu's avatar
Deshui Yu committed
259
            this.nniManager.getMetricData(req.params.job_id, req.query.type).then((metricsData: MetricDataRecord[]) => {
260
261
                res.send(metricsData);
            }).catch((err: Error) => {
chicm-ms's avatar
chicm-ms committed
262
                this.handleError(err, res);
263
264
265
266
267
268
269
270
271
272
273
            });
        });
    }

    private getMetricDataByRange(router: Router): void {
        router.get('/metric-data-range/:min_seq_id/:max_seq_id', async (req: Request, res: Response) => {
            const minSeqId = Number(req.params.min_seq_id);
            const maxSeqId = Number(req.params.max_seq_id);
            this.nniManager.getMetricDataByRange(minSeqId, maxSeqId).then((metricsData: MetricDataRecord[]) => {
                res.send(metricsData);
            }).catch((err: Error) => {
chicm-ms's avatar
chicm-ms committed
274
                this.handleError(err, res);
275
276
277
278
279
280
281
            });
        });
    }

    private getLatestMetricData(router: Router): void {
        router.get('/metric-data-latest/', async (req: Request, res: Response) => {
            this.nniManager.getLatestMetricData().then((metricsData: MetricDataRecord[]) => {
Deshui Yu's avatar
Deshui Yu committed
282
283
                res.send(metricsData);
            }).catch((err: Error) => {
chicm-ms's avatar
chicm-ms committed
284
                this.handleError(err, res);
Deshui Yu's avatar
Deshui Yu committed
285
286
287
288
            });
        });
    }

289
290
291
292
293
294
295
296
297
298
299
300
301
    private getTrialLog(router: Router): void {
        router.get('/trial-log/:id/:type', async(req: Request, res: Response) => {
            this.nniManager.getTrialLog(req.params.id, req.params.type).then((log: string) => {
                if (log === '') {
                    log = 'No logs available.'
                }
                res.send(log);
            }).catch((err: Error) => {
                this.handleError(err, res);
            });
        });
    }

302
303
304
305
306
    private exportData(router: Router): void {
        router.get('/export-data', (req: Request, res: Response) => {
            this.nniManager.exportData().then((exportedData: string) => {
                res.send(exportedData);
            }).catch((err: Error) => {
chicm-ms's avatar
chicm-ms committed
307
                this.handleError(err, res);
308
309
310
311
            });
        });
    }

312
313
314
315
316
317
318
319
320
321
    private getExperimentsInfo(router: Router): void {
        router.get('/experiments-info', (req: Request, res: Response) => {
            this.experimentsManager.getExperimentsInfo().then((experimentInfo: JSON) => {
                res.send(JSON.stringify(experimentInfo));
            }).catch((err: Error) => {
                this.handleError(err, res);
            });
        });
    }

Deshui Yu's avatar
Deshui Yu committed
322
323
324
325
    private setErrorPathForFailedJob(jobInfo: TrialJobInfo): TrialJobInfo {
        if (jobInfo === undefined || jobInfo.status !== 'FAILED' || jobInfo.logPath === undefined) {
            return jobInfo;
        }
chicm-ms's avatar
chicm-ms committed
326
        jobInfo.stderrPath = path.join(jobInfo.logPath, 'stderr');
Deshui Yu's avatar
Deshui Yu committed
327
328
329

        return jobInfo;
    }
330
331
332
333
334
335
336
337

    private setMessageforJob(jobInfo: TrialJobInfo): TrialJobInfo {
        if (jobInfo === undefined){
            return jobInfo
        }
        jobInfo.message = this.nniManager.getTrialJobMessage(jobInfo.trialJobId);
        return jobInfo
    }
Deshui Yu's avatar
Deshui Yu committed
338
339
}

340
export function createRestHandler(rs: NNIRestServer): Router {
Deshui Yu's avatar
Deshui Yu committed
341
342
343
344
    const handler: NNIRestHandler = new NNIRestHandler(rs);

    return handler.createRestHandler();
}