main.ts 7.77 KB
Newer Older
liuzhe-lz's avatar
liuzhe-lz committed
1
2
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
Deshui Yu's avatar
Deshui Yu committed
3
4
5
6
7

'use strict';

import { Container, Scope } from 'typescript-ioc';

8
import * as fs from 'fs';
9
import * as path from 'path';
10
import * as component from './common/component';
Deshui Yu's avatar
Deshui Yu committed
11
12
import { Database, DataStore } from './common/datastore';
import { setExperimentStartupInfo } from './common/experimentStartupInfo';
chicm-ms's avatar
chicm-ms committed
13
import { getLogger, Logger, logLevelNameMap } from './common/log';
SparkSnail's avatar
SparkSnail committed
14
import { Manager, ExperimentStartUpMode } from './common/manager';
Deshui Yu's avatar
Deshui Yu committed
15
import { TrainingService } from './common/trainingService';
16
import { getLogDir, mkDirP, parseArg, uniqueString } from './common/utils';
Deshui Yu's avatar
Deshui Yu committed
17
18
19
import { NNIDataStore } from './core/nniDataStore';
import { NNIManager } from './core/nnimanager';
import { SqlDB } from './core/sqlDatabase';
20
import { NNIRestServer } from './rest_server/nniRestServer';
21
22
23
import { FrameworkControllerTrainingService } from './training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService';
import { KubeflowTrainingService } from './training_service/kubernetes/kubeflow/kubeflowTrainingService';
import { LocalTrainingService } from './training_service/local/localTrainingService';
24
import { RouterTrainingService } from './training_service/reusable/routerTrainingService';
25
import { PAIYarnTrainingService } from './training_service/pai/paiYarn/paiYarnTrainingService';
Deshui Yu's avatar
Deshui Yu committed
26
27
28
import {
    RemoteMachineTrainingService
} from './training_service/remote_machine/remoteMachineTrainingService';
George Cheng's avatar
George Cheng committed
29
import { DLTSTrainingService } from './training_service/dlts/dltsTrainingService';
Deshui Yu's avatar
Deshui Yu committed
30

31
function initStartupInfo(
32
    startExpMode: string, resumeExperimentId: string, basePort: number, platform: string,
SparkSnail's avatar
SparkSnail committed
33
34
    logDirectory: string, experimentLogLevel: string, readonly: boolean): void {
    const createNew: boolean = (startExpMode === ExperimentStartUpMode.NEW);
Deshui Yu's avatar
Deshui Yu committed
35
    const expId: string = createNew ? uniqueString(8) : resumeExperimentId;
36
    setExperimentStartupInfo(createNew, expId, basePort, platform, logDirectory, experimentLogLevel, readonly);
Deshui Yu's avatar
Deshui Yu committed
37
38
}

39
async function initContainer(foreground: boolean, platformMode: string, logFileName?: string): Promise<void> {
Deshui Yu's avatar
Deshui Yu committed
40
    if (platformMode === 'local') {
41
42
43
        Container.bind(TrainingService)
            .to(LocalTrainingService)
            .scope(Scope.Singleton);
Deshui Yu's avatar
Deshui Yu committed
44
    } else if (platformMode === 'remote') {
45
46
47
        Container.bind(TrainingService)
            .to(RemoteMachineTrainingService)
            .scope(Scope.Singleton);
48
    } else if (platformMode === 'pai') {
49
        Container.bind(TrainingService)
50
            .to(RouterTrainingService)
51
52
            .scope(Scope.Singleton);
    } else if (platformMode === 'paiYarn') {
53
        Container.bind(TrainingService)
54
            .to(PAIYarnTrainingService)
55
            .scope(Scope.Singleton);
56
    } else if (platformMode === 'kubeflow') {
57
58
59
        Container.bind(TrainingService)
            .to(KubeflowTrainingService)
            .scope(Scope.Singleton);
60
    } else if (platformMode === 'frameworkcontroller') {
61
62
63
        Container.bind(TrainingService)
            .to(FrameworkControllerTrainingService)
            .scope(Scope.Singleton);
George Cheng's avatar
George Cheng committed
64
65
66
67
    } else if (platformMode === 'dlts') {
        Container.bind(TrainingService)
            .to(DLTSTrainingService)
            .scope(Scope.Singleton);
SparkSnail's avatar
SparkSnail committed
68
69
70
71
    } else if (platformMode === 'aml') {
        Container.bind(TrainingService)
            .to(RouterTrainingService)
            .scope(Scope.Singleton);
72
    } else {
chicm-ms's avatar
chicm-ms committed
73
        throw new Error(`Error: unsupported mode: ${platformMode}`);
Deshui Yu's avatar
Deshui Yu committed
74
    }
75
76
77
78
79
80
81
82
83
    Container.bind(Manager)
        .to(NNIManager)
        .scope(Scope.Singleton);
    Container.bind(Database)
        .to(SqlDB)
        .scope(Scope.Singleton);
    Container.bind(DataStore)
        .to(NNIDataStore)
        .scope(Scope.Singleton);
84
85
86
87
88
89
    const DEFAULT_LOGFILE: string = path.join(getLogDir(), 'nnimanager.log');
    if (foreground) {
        logFileName = undefined;
    } else if (logFileName === undefined) {
        logFileName = DEFAULT_LOGFILE;
    }
SparkSnail's avatar
SparkSnail committed
90
91
92
    Container.bind(Logger).provider({
        get: (): Logger => new Logger(logFileName)
    });
Deshui Yu's avatar
Deshui Yu committed
93
94
95
96
97
98
    const ds: DataStore = component.get(DataStore);

    await ds.init();
}

function usage(): void {
99
    console.info('usage: node main.js --port <port> --mode \
SparkSnail's avatar
SparkSnail committed
100
    <local/remote/pai/kubeflow/frameworkcontroller/paiYarn/aml> --start_mode <new/resume> --experiment_id <id> --foreground <true/false>');
Deshui Yu's avatar
Deshui Yu committed
101
102
103
}

const strPort: string = parseArg(['--port', '-p']);
goooxu's avatar
goooxu committed
104
105
106
if (!strPort || strPort.length === 0) {
    usage();
    process.exit(1);
Deshui Yu's avatar
Deshui Yu committed
107
108
}

109
110
111
112
113
114
115
116
const foregroundArg: string = parseArg(['--foreground', '-f']);
if (!('true' || 'false').includes(foregroundArg.toLowerCase())) {
    console.log(`FATAL: foreground property should only be true or false`);
    usage();
    process.exit(1);
}
const foreground: boolean = foregroundArg.toLowerCase() === 'true' ? true : false;

goooxu's avatar
goooxu committed
117
118
const port: number = parseInt(strPort, 10);

Deshui Yu's avatar
Deshui Yu committed
119
const mode: string = parseArg(['--mode', '-m']);
SparkSnail's avatar
SparkSnail committed
120
if (!['local', 'remote', 'pai', 'kubeflow', 'frameworkcontroller', 'paiYarn', 'dlts', 'aml'].includes(mode)) {
121
    console.log(`FATAL: unknown mode: ${mode}`);
Deshui Yu's avatar
Deshui Yu committed
122
123
124
125
126
    usage();
    process.exit(1);
}

const startMode: string = parseArg(['--start_mode', '-s']);
SparkSnail's avatar
SparkSnail committed
127
if (![ExperimentStartUpMode.NEW, ExperimentStartUpMode.RESUME].includes(startMode)) {
128
    console.log(`FATAL: unknown start_mode: ${startMode}`);
Deshui Yu's avatar
Deshui Yu committed
129
130
131
132
133
    usage();
    process.exit(1);
}

const experimentId: string = parseArg(['--experiment_id', '-id']);
SparkSnail's avatar
SparkSnail committed
134
135
if ((startMode === ExperimentStartUpMode.RESUME) && experimentId.trim().length < 1) {
    console.log(`FATAL: cannot resume the experiment, invalid experiment_id: ${experimentId}`);
Deshui Yu's avatar
Deshui Yu committed
136
137
138
139
    usage();
    process.exit(1);
}

140
141
142
143
144
145
146
147
const logDir: string = parseArg(['--log_dir', '-ld']);
if (logDir.length > 0) {
    if (!fs.existsSync(logDir)) {
        console.log(`FATAL: log_dir ${logDir} does not exist`);
    }
}

const logLevel: string = parseArg(['--log_level', '-ll']);
chicm-ms's avatar
chicm-ms committed
148
if (logLevel.length > 0 && !logLevelNameMap.has(logLevel)) {
149
150
151
    console.log(`FATAL: invalid log_level: ${logLevel}`);
}

SparkSnail's avatar
SparkSnail committed
152
153
154
155
156
157
158
159
const readonlyArg: string = parseArg(['--readonly', '-r']);
if (!('true' || 'false').includes(readonlyArg.toLowerCase())) {
    console.log(`FATAL: readonly property should only be true or false`);
    usage();
    process.exit(1);
}
const readonly = readonlyArg.toLowerCase() == 'true' ? true : false;

160
initStartupInfo(startMode, experimentId, port, mode, logDir, logLevel, readonly);
Deshui Yu's avatar
Deshui Yu committed
161

162
163
mkDirP(getLogDir())
    .then(async () => {
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
        try {
            await initContainer(foreground, mode);
            const restServer: NNIRestServer = component.get(NNIRestServer);
            await restServer.start();
            const log: Logger = getLogger();
            log.info(`Rest server listening on: ${restServer.endPoint}`);
        } catch (err) {
            const log: Logger = getLogger();
            log.error(`${err.stack}`);
            throw err;
        }
    })
    .catch((err: Error) => {
        console.error(`Failed to create log dir: ${err.stack}`);
    });
179

180
181
182
183
function getStopSignal(): any {
    if (process.platform === "win32") {
        return 'SIGBREAK';
    }
184
    else {
185
186
187
188
        return 'SIGTERM';
    }
}

189
190
191
192
193
194
195
196
197
function getCtrlCSignal(): any {
    return 'SIGINT';
}

process.on(getCtrlCSignal(), async () => {
    const log: Logger = getLogger();
    log.info(`Get SIGINT signal!`);
});

198
process.on(getStopSignal(), async () => {
199
    const log: Logger = getLogger();
SparkSnail's avatar
SparkSnail committed
200
    let hasError: boolean = false;
201
    try {
SparkSnail's avatar
SparkSnail committed
202
203
204
205
206
207
        const nniManager: Manager = component.get(Manager);
        await nniManager.stopExperiment();
        const ds: DataStore = component.get(DataStore);
        await ds.close();
        const restServer: NNIRestServer = component.get(NNIRestServer);
        await restServer.stop();
208
    } catch (err) {
SparkSnail's avatar
SparkSnail committed
209
210
        hasError = true;
        log.error(`${err.stack}`);
211
    } finally {
SparkSnail's avatar
SparkSnail committed
212
        await log.close();
213
        process.exit(hasError ? 1 : 0);
SparkSnail's avatar
SparkSnail committed
214
    }
215
});