main.ts 7.81 KB
Newer Older
liuzhe-lz's avatar
liuzhe-lz committed
1
2
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
Deshui Yu's avatar
Deshui Yu committed
3
4
5
6
7

'use strict';

import { Container, Scope } from 'typescript-ioc';

8
import * as fs from 'fs';
9
import * as path from 'path';
10
import * as component from './common/component';
Deshui Yu's avatar
Deshui Yu committed
11
12
import { Database, DataStore } from './common/datastore';
import { setExperimentStartupInfo } from './common/experimentStartupInfo';
chicm-ms's avatar
chicm-ms committed
13
import { getLogger, Logger, logLevelNameMap } from './common/log';
SparkSnail's avatar
SparkSnail committed
14
import { Manager, ExperimentStartUpMode } from './common/manager';
Deshui Yu's avatar
Deshui Yu committed
15
import { TrainingService } from './common/trainingService';
16
import { getLogDir, mkDirP, parseArg, uniqueString } from './common/utils';
Deshui Yu's avatar
Deshui Yu committed
17
18
19
import { NNIDataStore } from './core/nniDataStore';
import { NNIManager } from './core/nnimanager';
import { SqlDB } from './core/sqlDatabase';
20
import { NNIRestServer } from './rest_server/nniRestServer';
21
import { FrameworkControllerTrainingService } from './training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService';
22
import { AdlTrainingService } from './training_service/kubernetes/adl/adlTrainingService';
23
24
import { KubeflowTrainingService } from './training_service/kubernetes/kubeflow/kubeflowTrainingService';
import { LocalTrainingService } from './training_service/local/localTrainingService';
25
import { RouterTrainingService } from './training_service/reusable/routerTrainingService';
26
import { PAIYarnTrainingService } from './training_service/pai/paiYarn/paiYarnTrainingService';
George Cheng's avatar
George Cheng committed
27
import { DLTSTrainingService } from './training_service/dlts/dltsTrainingService';
Deshui Yu's avatar
Deshui Yu committed
28

29
function initStartupInfo(
30
    startExpMode: string, resumeExperimentId: string, basePort: number, platform: string,
SparkSnail's avatar
SparkSnail committed
31
32
    logDirectory: string, experimentLogLevel: string, readonly: boolean): void {
    const createNew: boolean = (startExpMode === ExperimentStartUpMode.NEW);
Deshui Yu's avatar
Deshui Yu committed
33
    const expId: string = createNew ? uniqueString(8) : resumeExperimentId;
34
    setExperimentStartupInfo(createNew, expId, basePort, platform, logDirectory, experimentLogLevel, readonly);
Deshui Yu's avatar
Deshui Yu committed
35
36
}

37
async function initContainer(foreground: boolean, platformMode: string, logFileName?: string): Promise<void> {
38
39
40
41
42
    if (platformMode === 'adl') {
        Container.bind(TrainingService)
            .to(AdlTrainingService)
            .scope(Scope.Singleton);
    } else if (platformMode === 'local') {
43
44
45
        Container.bind(TrainingService)
            .to(LocalTrainingService)
            .scope(Scope.Singleton);
Deshui Yu's avatar
Deshui Yu committed
46
    } else if (platformMode === 'remote') {
47
        Container.bind(TrainingService)
48
            .to(RouterTrainingService)
49
            .scope(Scope.Singleton);
50
    } else if (platformMode === 'pai') {
51
        Container.bind(TrainingService)
52
            .to(RouterTrainingService)
53
54
            .scope(Scope.Singleton);
    } else if (platformMode === 'paiYarn') {
55
        Container.bind(TrainingService)
56
            .to(PAIYarnTrainingService)
57
            .scope(Scope.Singleton);
58
    } else if (platformMode === 'kubeflow') {
59
60
61
        Container.bind(TrainingService)
            .to(KubeflowTrainingService)
            .scope(Scope.Singleton);
62
    } else if (platformMode === 'frameworkcontroller') {
63
64
65
        Container.bind(TrainingService)
            .to(FrameworkControllerTrainingService)
            .scope(Scope.Singleton);
George Cheng's avatar
George Cheng committed
66
67
68
69
    } else if (platformMode === 'dlts') {
        Container.bind(TrainingService)
            .to(DLTSTrainingService)
            .scope(Scope.Singleton);
SparkSnail's avatar
SparkSnail committed
70
71
72
73
    } else if (platformMode === 'aml') {
        Container.bind(TrainingService)
            .to(RouterTrainingService)
            .scope(Scope.Singleton);
74
    } else {
chicm-ms's avatar
chicm-ms committed
75
        throw new Error(`Error: unsupported mode: ${platformMode}`);
Deshui Yu's avatar
Deshui Yu committed
76
    }
77
78
79
80
81
82
83
84
85
    Container.bind(Manager)
        .to(NNIManager)
        .scope(Scope.Singleton);
    Container.bind(Database)
        .to(SqlDB)
        .scope(Scope.Singleton);
    Container.bind(DataStore)
        .to(NNIDataStore)
        .scope(Scope.Singleton);
86
87
88
89
90
91
    const DEFAULT_LOGFILE: string = path.join(getLogDir(), 'nnimanager.log');
    if (foreground) {
        logFileName = undefined;
    } else if (logFileName === undefined) {
        logFileName = DEFAULT_LOGFILE;
    }
SparkSnail's avatar
SparkSnail committed
92
93
94
    Container.bind(Logger).provider({
        get: (): Logger => new Logger(logFileName)
    });
Deshui Yu's avatar
Deshui Yu committed
95
96
97
98
99
100
    const ds: DataStore = component.get(DataStore);

    await ds.init();
}

function usage(): void {
101
    console.info('usage: node main.js --port <port> --mode \
102
    <adl/local/remote/pai/kubeflow/frameworkcontroller/paiYarn/aml> --start_mode <new/resume> --experiment_id <id> --foreground <true/false>');
Deshui Yu's avatar
Deshui Yu committed
103
104
105
}

const strPort: string = parseArg(['--port', '-p']);
goooxu's avatar
goooxu committed
106
107
108
if (!strPort || strPort.length === 0) {
    usage();
    process.exit(1);
Deshui Yu's avatar
Deshui Yu committed
109
110
}

111
112
113
114
115
116
117
118
const foregroundArg: string = parseArg(['--foreground', '-f']);
if (!('true' || 'false').includes(foregroundArg.toLowerCase())) {
    console.log(`FATAL: foreground property should only be true or false`);
    usage();
    process.exit(1);
}
const foreground: boolean = foregroundArg.toLowerCase() === 'true' ? true : false;

goooxu's avatar
goooxu committed
119
120
const port: number = parseInt(strPort, 10);

Deshui Yu's avatar
Deshui Yu committed
121
const mode: string = parseArg(['--mode', '-m']);
122
if (!['adl', 'local', 'remote', 'pai', 'kubeflow', 'frameworkcontroller', 'paiYarn', 'dlts', 'aml'].includes(mode)) {
123
    console.log(`FATAL: unknown mode: ${mode}`);
Deshui Yu's avatar
Deshui Yu committed
124
125
126
127
128
    usage();
    process.exit(1);
}

const startMode: string = parseArg(['--start_mode', '-s']);
SparkSnail's avatar
SparkSnail committed
129
if (![ExperimentStartUpMode.NEW, ExperimentStartUpMode.RESUME].includes(startMode)) {
130
    console.log(`FATAL: unknown start_mode: ${startMode}`);
Deshui Yu's avatar
Deshui Yu committed
131
132
133
134
135
    usage();
    process.exit(1);
}

const experimentId: string = parseArg(['--experiment_id', '-id']);
SparkSnail's avatar
SparkSnail committed
136
137
if ((startMode === ExperimentStartUpMode.RESUME) && experimentId.trim().length < 1) {
    console.log(`FATAL: cannot resume the experiment, invalid experiment_id: ${experimentId}`);
Deshui Yu's avatar
Deshui Yu committed
138
139
140
141
    usage();
    process.exit(1);
}

142
143
144
145
146
147
148
149
const logDir: string = parseArg(['--log_dir', '-ld']);
if (logDir.length > 0) {
    if (!fs.existsSync(logDir)) {
        console.log(`FATAL: log_dir ${logDir} does not exist`);
    }
}

const logLevel: string = parseArg(['--log_level', '-ll']);
chicm-ms's avatar
chicm-ms committed
150
if (logLevel.length > 0 && !logLevelNameMap.has(logLevel)) {
151
152
153
    console.log(`FATAL: invalid log_level: ${logLevel}`);
}

SparkSnail's avatar
SparkSnail committed
154
155
156
157
158
159
160
161
const readonlyArg: string = parseArg(['--readonly', '-r']);
if (!('true' || 'false').includes(readonlyArg.toLowerCase())) {
    console.log(`FATAL: readonly property should only be true or false`);
    usage();
    process.exit(1);
}
const readonly = readonlyArg.toLowerCase() == 'true' ? true : false;

162
initStartupInfo(startMode, experimentId, port, mode, logDir, logLevel, readonly);
Deshui Yu's avatar
Deshui Yu committed
163

164
165
mkDirP(getLogDir())
    .then(async () => {
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
        try {
            await initContainer(foreground, mode);
            const restServer: NNIRestServer = component.get(NNIRestServer);
            await restServer.start();
            const log: Logger = getLogger();
            log.info(`Rest server listening on: ${restServer.endPoint}`);
        } catch (err) {
            const log: Logger = getLogger();
            log.error(`${err.stack}`);
            throw err;
        }
    })
    .catch((err: Error) => {
        console.error(`Failed to create log dir: ${err.stack}`);
    });
181

182
function getStopSignal(): any {
liuzhe-lz's avatar
liuzhe-lz committed
183
    return 'SIGTERM';
184
185
}

186
187
188
189
190
191
192
193
194
function getCtrlCSignal(): any {
    return 'SIGINT';
}

process.on(getCtrlCSignal(), async () => {
    const log: Logger = getLogger();
    log.info(`Get SIGINT signal!`);
});

195
process.on(getStopSignal(), async () => {
196
    const log: Logger = getLogger();
SparkSnail's avatar
SparkSnail committed
197
    let hasError: boolean = false;
198
    try {
SparkSnail's avatar
SparkSnail committed
199
200
201
202
203
204
        const nniManager: Manager = component.get(Manager);
        await nniManager.stopExperiment();
        const ds: DataStore = component.get(DataStore);
        await ds.close();
        const restServer: NNIRestServer = component.get(NNIRestServer);
        await restServer.stop();
205
    } catch (err) {
SparkSnail's avatar
SparkSnail committed
206
207
        hasError = true;
        log.error(`${err.stack}`);
208
    } finally {
SparkSnail's avatar
SparkSnail committed
209
        await log.close();
210
        process.exit(hasError ? 1 : 0);
SparkSnail's avatar
SparkSnail committed
211
    }
212
});