main.ts 7.61 KB
Newer Older
liuzhe-lz's avatar
liuzhe-lz committed
1
2
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
Deshui Yu's avatar
Deshui Yu committed
3
4
5
6
7

'use strict';

import { Container, Scope } from 'typescript-ioc';

8
import * as fs from 'fs';
9
import * as path from 'path';
10
import * as component from './common/component';
Deshui Yu's avatar
Deshui Yu committed
11
12
import { Database, DataStore } from './common/datastore';
import { setExperimentStartupInfo } from './common/experimentStartupInfo';
chicm-ms's avatar
chicm-ms committed
13
import { getLogger, Logger, logLevelNameMap } from './common/log';
SparkSnail's avatar
SparkSnail committed
14
import { Manager, ExperimentStartUpMode } from './common/manager';
Deshui Yu's avatar
Deshui Yu committed
15
import { TrainingService } from './common/trainingService';
16
import { getLogDir, mkDirP, parseArg, uniqueString } from './common/utils';
Deshui Yu's avatar
Deshui Yu committed
17
18
19
import { NNIDataStore } from './core/nniDataStore';
import { NNIManager } from './core/nnimanager';
import { SqlDB } from './core/sqlDatabase';
20
import { NNIRestServer } from './rest_server/nniRestServer';
21
22
23
import { FrameworkControllerTrainingService } from './training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService';
import { KubeflowTrainingService } from './training_service/kubernetes/kubeflow/kubeflowTrainingService';
import { LocalTrainingService } from './training_service/local/localTrainingService';
24
import { RouterTrainingService } from './training_service/reusable/routerTrainingService';
25
import { PAIYarnTrainingService } from './training_service/pai/paiYarn/paiYarnTrainingService';
Deshui Yu's avatar
Deshui Yu committed
26
27
28
import {
    RemoteMachineTrainingService
} from './training_service/remote_machine/remoteMachineTrainingService';
George Cheng's avatar
George Cheng committed
29
import { DLTSTrainingService } from './training_service/dlts/dltsTrainingService';
Deshui Yu's avatar
Deshui Yu committed
30

31
function initStartupInfo(
32
    startExpMode: string, resumeExperimentId: string, basePort: number, platform: string,
SparkSnail's avatar
SparkSnail committed
33
34
    logDirectory: string, experimentLogLevel: string, readonly: boolean): void {
    const createNew: boolean = (startExpMode === ExperimentStartUpMode.NEW);
Deshui Yu's avatar
Deshui Yu committed
35
    const expId: string = createNew ? uniqueString(8) : resumeExperimentId;
36
    setExperimentStartupInfo(createNew, expId, basePort, platform, logDirectory, experimentLogLevel, readonly);
Deshui Yu's avatar
Deshui Yu committed
37
38
}

39
async function initContainer(foreground: boolean, platformMode: string, logFileName?: string): Promise<void> {
Deshui Yu's avatar
Deshui Yu committed
40
    if (platformMode === 'local') {
41
42
43
        Container.bind(TrainingService)
            .to(LocalTrainingService)
            .scope(Scope.Singleton);
Deshui Yu's avatar
Deshui Yu committed
44
    } else if (platformMode === 'remote') {
45
46
47
        Container.bind(TrainingService)
            .to(RemoteMachineTrainingService)
            .scope(Scope.Singleton);
48
    } else if (platformMode === 'pai') {
49
        Container.bind(TrainingService)
50
            .to(RouterTrainingService)
51
52
            .scope(Scope.Singleton);
    } else if (platformMode === 'paiYarn') {
53
        Container.bind(TrainingService)
54
            .to(PAIYarnTrainingService)
55
            .scope(Scope.Singleton);
56
    } else if (platformMode === 'kubeflow') {
57
58
59
        Container.bind(TrainingService)
            .to(KubeflowTrainingService)
            .scope(Scope.Singleton);
60
    } else if (platformMode === 'frameworkcontroller') {
61
62
63
        Container.bind(TrainingService)
            .to(FrameworkControllerTrainingService)
            .scope(Scope.Singleton);
George Cheng's avatar
George Cheng committed
64
65
66
67
    } else if (platformMode === 'dlts') {
        Container.bind(TrainingService)
            .to(DLTSTrainingService)
            .scope(Scope.Singleton);
68
    } else {
chicm-ms's avatar
chicm-ms committed
69
        throw new Error(`Error: unsupported mode: ${platformMode}`);
Deshui Yu's avatar
Deshui Yu committed
70
    }
71
72
73
74
75
76
77
78
79
    Container.bind(Manager)
        .to(NNIManager)
        .scope(Scope.Singleton);
    Container.bind(Database)
        .to(SqlDB)
        .scope(Scope.Singleton);
    Container.bind(DataStore)
        .to(NNIDataStore)
        .scope(Scope.Singleton);
80
81
82
83
84
85
    const DEFAULT_LOGFILE: string = path.join(getLogDir(), 'nnimanager.log');
    if (foreground) {
        logFileName = undefined;
    } else if (logFileName === undefined) {
        logFileName = DEFAULT_LOGFILE;
    }
SparkSnail's avatar
SparkSnail committed
86
87
88
    Container.bind(Logger).provider({
        get: (): Logger => new Logger(logFileName)
    });
Deshui Yu's avatar
Deshui Yu committed
89
90
91
92
93
94
    const ds: DataStore = component.get(DataStore);

    await ds.init();
}

function usage(): void {
95
    console.info('usage: node main.js --port <port> --mode \
96
    <local/remote/pai/kubeflow/frameworkcontroller/paiYarn> --start_mode <new/resume> --experiment_id <id> --foreground <true/false>');
Deshui Yu's avatar
Deshui Yu committed
97
98
99
}

const strPort: string = parseArg(['--port', '-p']);
goooxu's avatar
goooxu committed
100
101
102
if (!strPort || strPort.length === 0) {
    usage();
    process.exit(1);
Deshui Yu's avatar
Deshui Yu committed
103
104
}

105
106
107
108
109
110
111
112
const foregroundArg: string = parseArg(['--foreground', '-f']);
if (!('true' || 'false').includes(foregroundArg.toLowerCase())) {
    console.log(`FATAL: foreground property should only be true or false`);
    usage();
    process.exit(1);
}
const foreground: boolean = foregroundArg.toLowerCase() === 'true' ? true : false;

goooxu's avatar
goooxu committed
113
114
const port: number = parseInt(strPort, 10);

Deshui Yu's avatar
Deshui Yu committed
115
const mode: string = parseArg(['--mode', '-m']);
George Cheng's avatar
George Cheng committed
116
if (!['local', 'remote', 'pai', 'kubeflow', 'frameworkcontroller', 'paiYarn', 'dlts'].includes(mode)) {
117
    console.log(`FATAL: unknown mode: ${mode}`);
Deshui Yu's avatar
Deshui Yu committed
118
119
120
121
122
    usage();
    process.exit(1);
}

const startMode: string = parseArg(['--start_mode', '-s']);
SparkSnail's avatar
SparkSnail committed
123
if (![ExperimentStartUpMode.NEW, ExperimentStartUpMode.RESUME].includes(startMode)) {
124
    console.log(`FATAL: unknown start_mode: ${startMode}`);
Deshui Yu's avatar
Deshui Yu committed
125
126
127
128
129
    usage();
    process.exit(1);
}

const experimentId: string = parseArg(['--experiment_id', '-id']);
SparkSnail's avatar
SparkSnail committed
130
131
if ((startMode === ExperimentStartUpMode.RESUME) && experimentId.trim().length < 1) {
    console.log(`FATAL: cannot resume the experiment, invalid experiment_id: ${experimentId}`);
Deshui Yu's avatar
Deshui Yu committed
132
133
134
135
    usage();
    process.exit(1);
}

136
137
138
139
140
141
142
143
const logDir: string = parseArg(['--log_dir', '-ld']);
if (logDir.length > 0) {
    if (!fs.existsSync(logDir)) {
        console.log(`FATAL: log_dir ${logDir} does not exist`);
    }
}

const logLevel: string = parseArg(['--log_level', '-ll']);
chicm-ms's avatar
chicm-ms committed
144
if (logLevel.length > 0 && !logLevelNameMap.has(logLevel)) {
145
146
147
    console.log(`FATAL: invalid log_level: ${logLevel}`);
}

SparkSnail's avatar
SparkSnail committed
148
149
150
151
152
153
154
155
const readonlyArg: string = parseArg(['--readonly', '-r']);
if (!('true' || 'false').includes(readonlyArg.toLowerCase())) {
    console.log(`FATAL: readonly property should only be true or false`);
    usage();
    process.exit(1);
}
const readonly = readonlyArg.toLowerCase() == 'true' ? true : false;

156
initStartupInfo(startMode, experimentId, port, mode, logDir, logLevel, readonly);
Deshui Yu's avatar
Deshui Yu committed
157

158
159
mkDirP(getLogDir())
    .then(async () => {
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
        try {
            await initContainer(foreground, mode);
            const restServer: NNIRestServer = component.get(NNIRestServer);
            await restServer.start();
            const log: Logger = getLogger();
            log.info(`Rest server listening on: ${restServer.endPoint}`);
        } catch (err) {
            const log: Logger = getLogger();
            log.error(`${err.stack}`);
            throw err;
        }
    })
    .catch((err: Error) => {
        console.error(`Failed to create log dir: ${err.stack}`);
    });
175

176
177
178
179
function getStopSignal(): any {
    if (process.platform === "win32") {
        return 'SIGBREAK';
    }
180
    else {
181
182
183
184
        return 'SIGTERM';
    }
}

185
186
187
188
189
190
191
192
193
function getCtrlCSignal(): any {
    return 'SIGINT';
}

process.on(getCtrlCSignal(), async () => {
    const log: Logger = getLogger();
    log.info(`Get SIGINT signal!`);
});

194
process.on(getStopSignal(), async () => {
195
    const log: Logger = getLogger();
SparkSnail's avatar
SparkSnail committed
196
    let hasError: boolean = false;
197
    try {
SparkSnail's avatar
SparkSnail committed
198
199
200
201
202
203
        const nniManager: Manager = component.get(Manager);
        await nniManager.stopExperiment();
        const ds: DataStore = component.get(DataStore);
        await ds.close();
        const restServer: NNIRestServer = component.get(NNIRestServer);
        await restServer.stop();
204
    } catch (err) {
SparkSnail's avatar
SparkSnail committed
205
206
        hasError = true;
        log.error(`${err.stack}`);
207
    } finally {
SparkSnail's avatar
SparkSnail committed
208
        await log.close();
209
        process.exit(hasError ? 1 : 0);
SparkSnail's avatar
SparkSnail committed
210
    }
211
});