utils.ts 11.8 KB
Newer Older
liuzhe-lz's avatar
liuzhe-lz committed
1
2
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
Deshui Yu's avatar
Deshui Yu committed
3
4
5

'use strict';

6
import * as assert from 'assert';
Deshui Yu's avatar
Deshui Yu committed
7
import { randomBytes } from 'crypto';
8
import * as cpp from 'child-process-promise';
9
10
import * as cp from 'child_process';
import { ChildProcess, spawn, StdioOptions } from 'child_process';
Deshui Yu's avatar
Deshui Yu committed
11
12
13
14
15
16
17
18
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { Deferred } from 'ts-deferred';
import { Container } from 'typescript-ioc';
import * as util from 'util';

import { Database, DataStore } from './datastore';
chicm-ms's avatar
chicm-ms committed
19
import { ExperimentStartupInfo, getExperimentStartupInfo, setExperimentStartupInfo } from './experimentStartupInfo';
chicm-ms's avatar
chicm-ms committed
20
import { ExperimentParams, Manager } from './manager';
QuanluZhang's avatar
QuanluZhang committed
21
import { HyperParameters, TrainingService, TrialJobStatus } from './trainingService';
Deshui Yu's avatar
Deshui Yu committed
22

23
function getExperimentRootDir(): string {
24
    return getExperimentStartupInfo()
25
        .getLogDir();
Deshui Yu's avatar
Deshui Yu committed
26
27
}

28
function getLogDir(): string {
Deshui Yu's avatar
Deshui Yu committed
29
30
31
    return path.join(getExperimentRootDir(), 'log');
}

32
function getLogLevel(): string {
33
    return getExperimentStartupInfo()
34
        .getLogLevel();
35
36
}

Deshui Yu's avatar
Deshui Yu committed
37
38
39
40
function getDefaultDatabaseDir(): string {
    return path.join(getExperimentRootDir(), 'db');
}

QuanluZhang's avatar
QuanluZhang committed
41
42
43
44
function getCheckpointDir(): string {
    return path.join(getExperimentRootDir(), 'checkpoint');
}

Deshui Yu's avatar
Deshui Yu committed
45
46
47
48
49
50
51
52
function mkDirP(dirPath: string): Promise<void> {
    const deferred: Deferred<void> = new Deferred<void>();
    fs.exists(dirPath, (exists: boolean) => {
        if (exists) {
            deferred.resolve();
        } else {
            const parent: string = path.dirname(dirPath);
            mkDirP(parent).then(() => {
53
                fs.mkdir(dirPath, (err: Error) => {
Deshui Yu's avatar
Deshui Yu committed
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
                    if (err) {
                        deferred.reject(err);
                    } else {
                        deferred.resolve();
                    }
                });
            }).catch((err: Error) => {
                deferred.reject(err);
            });
        }
    });

    return deferred.promise;
}

function mkDirPSync(dirPath: string): void {
    if (fs.existsSync(dirPath)) {
        return;
    }
    mkDirPSync(path.dirname(dirPath));
    fs.mkdirSync(dirPath);
}

const delay: (ms: number) => Promise<void> = util.promisify(setTimeout);

/**
 * Convert index to character
 * @param index index
 * @returns a mapping character
 */
function charMap(index: number): number {
    if (index < 26) {
        return index + 97;
    } else if (index < 52) {
        return index - 26 + 65;
    } else {
        return index - 52 + 48;
    }
}

/**
 * Generate a unique string by length
 * @param len length of string
 * @returns a unique string
 */
function uniqueString(len: number): string {
    if (len === 0) {
        return '';
    }
    const byteLength: number = Math.ceil((Math.log2(52) + Math.log2(62) * (len - 1)) / 8);
    let num: number = randomBytes(byteLength).reduce((a: number, b: number) => a * 256 + b, 0);
    const codes: number[] = [];
    codes.push(charMap(num % 52));
    num = Math.floor(num / 52);
    for (let i: number = 1; i < len; i++) {
        codes.push(charMap(num % 62));
        num = Math.floor(num / 62);
    }

    return String.fromCharCode(...codes);
}

116
117
118
119
function randomInt(max: number): number {
    return Math.floor(Math.random() * max);
}

120
121
122
123
124
function randomSelect<T>(a: T[]): T {
    assert(a !== undefined);

    return a[Math.floor(Math.random() * a.length)];
}
125

Deshui Yu's avatar
Deshui Yu committed
126
127
128
129
130
131
132
133
134
135
136
137
function parseArg(names: string[]): string {
    if (process.argv.length >= 4) {
        for (let i: number = 2; i < process.argv.length - 1; i++) {
            if (names.includes(process.argv[i])) {
                return process.argv[i + 1];
            }
        }
    }

    return '';
}

138
function getCmdPy(): string {
139
    let cmd = 'python3';
140
    if (process.platform === 'win32') {
141
142
143
144
145
        cmd = 'python';
    }
    return cmd;
}

146
/**
147
 * Generate command line to start automl algorithm(s),
QuanluZhang's avatar
QuanluZhang committed
148
 * either start advisor or start a process which runs tuner and assessor
149
 *
chicm-ms's avatar
chicm-ms committed
150
 * @param expParams: experiment startup parameters
151
152
 *
 */
chicm-ms's avatar
chicm-ms committed
153
154
155
156
function getMsgDispatcherCommand(expParams: ExperimentParams): string {
    const clonedParams = Object.assign({}, expParams);
    delete clonedParams.searchSpace;
    return `${getCmdPy()} -m nni --exp_params ${Buffer.from(JSON.stringify(clonedParams)).toString('base64')}`;
157
158
}

159
160
161
162
/**
 * Generate parameter file name based on HyperParameters object
 * @param hyperParameters HyperParameters instance
 */
chicm-ms's avatar
chicm-ms committed
163
function generateParamFileName(hyperParameters: HyperParameters): string {
164
165
166
    assert(hyperParameters !== undefined);
    assert(hyperParameters.index >= 0);

chicm-ms's avatar
chicm-ms committed
167
    let paramFileName: string;
168
    if (hyperParameters.index == 0) {
169
170
171
172
173
174
175
        paramFileName = 'parameter.cfg';
    } else {
        paramFileName = `parameter_${hyperParameters.index}.cfg`
    }
    return paramFileName;
}

Deshui Yu's avatar
Deshui Yu committed
176
177
178
179
180
181
182
183
184
185
186
/**
 * Initialize a pseudo experiment environment for unit test.
 * Must be paired with `cleanupUnitTest()`.
 */
function prepareUnitTest(): void {
    Container.snapshot(ExperimentStartupInfo);
    Container.snapshot(Database);
    Container.snapshot(DataStore);
    Container.snapshot(TrainingService);
    Container.snapshot(Manager);

187
    setExperimentStartupInfo(true, 'unittest', 8080);
Deshui Yu's avatar
Deshui Yu committed
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
    mkDirPSync(getLogDir());

    const sqliteFile: string = path.join(getDefaultDatabaseDir(), 'nni.sqlite');
    try {
        fs.unlinkSync(sqliteFile);
    } catch (err) {
        // file not exists, good
    }
}

/**
 * Clean up unit test pseudo experiment.
 * Must be paired with `prepareUnitTest()`.
 */
function cleanupUnitTest(): void {
    Container.restore(Manager);
    Container.restore(TrainingService);
    Container.restore(DataStore);
    Container.restore(Database);
    Container.restore(ExperimentStartupInfo);
}

chicm-ms's avatar
chicm-ms committed
210
let cachedipv4Address: string = '';
211
212
213
214
/**
 * Get IPv4 address of current machine
 */
function getIPV4Address(): string {
215
216
217
    if (cachedipv4Address && cachedipv4Address.length > 0) {
        return cachedipv4Address;
    }
218

219
220
221
    if (os.networkInterfaces().eth0) {
        for (const item of os.networkInterfaces().eth0) {
            if (item.family === 'IPv4') {
222
223
224
                cachedipv4Address = item.address;
                return cachedipv4Address;
            }
225
        }
226
227
    } else {
        throw Error('getIPV4Address() failed because os.networkInterfaces().eth0 is undefined.');
228
    }
229
230

    throw Error('getIPV4Address() failed because no valid IPv4 address found.')
231
232
}

QuanluZhang's avatar
QuanluZhang committed
233
234
235
236
237
238
239
/**
 * Get the status of canceled jobs according to the hint isEarlyStopped
 */
function getJobCancelStatus(isEarlyStopped: boolean): TrialJobStatus {
    return isEarlyStopped ? 'EARLY_STOPPED' : 'USER_CANCELED';
}

240
241
242
243
/**
 * Utility method to calculate file numbers under a directory, recursively
 * @param directory directory name
 */
chicm-ms's avatar
chicm-ms committed
244
function countFilesRecursively(directory: string): Promise<number> {
245
    if (!fs.existsSync(directory)) {
246
247
248
249
250
        throw Error(`Direcotory ${directory} doesn't exist`);
    }

    const deferred: Deferred<number> = new Deferred<number>();

chicm-ms's avatar
chicm-ms committed
251
252
    let timeoutId: NodeJS.Timer
    const delayTimeout: Promise<number> = new Promise((resolve: Function, reject: Function): void => {
253
254
255
256
257
258
259
        // Set timeout and reject the promise once reach timeout (5 seconds)
        timeoutId = setTimeout(() => {
            reject(new Error(`Timeout: path ${directory} has too many files`));
        }, 5000);
    });

    let fileCount: number = -1;
260
    let cmd: string;
261
    if (process.platform === "win32") {
262
263
        cmd = `powershell "Get-ChildItem -Path ${directory} -Recurse -File | Measure-Object | %{$_.Count}"`
    } else {
264
        cmd = `find ${directory} -type f | wc -l`;
265
266
    }
    cpp.exec(cmd).then((result) => {
267
        if (result.stdout && parseInt(result.stdout)) {
268
            fileCount = parseInt(result.stdout);
269
270
271
272
273
274
275
276
        }
        deferred.resolve(fileCount);
    });
    return Promise.race([deferred.promise, delayTimeout]).finally(() => {
        clearTimeout(timeoutId);
    });
}

277
export function validateFileName(fileName: string): boolean {
chicm-ms's avatar
chicm-ms committed
278
    const pattern: string = '^[a-z0-9A-Z._-]+$';
279
    const validateResult = fileName.match(pattern);
280
    if (validateResult) {
281
282
283
284
285
286
        return true;
    }
    return false;
}

async function validateFileNameRecursively(directory: string): Promise<boolean> {
287
    if (!fs.existsSync(directory)) {
288
289
290
291
292
        throw Error(`Direcotory ${directory} doesn't exist`);
    }

    const fileNameArray: string[] = fs.readdirSync(directory);
    let result = true;
293
    for (const name of fileNameArray) {
294
295
296
297
298
299
300
        const fullFilePath: string = path.join(directory, name);
        try {
            // validate file names and directory names
            result = validateFileName(name);
            if (fs.lstatSync(fullFilePath).isDirectory()) {
                result = result && await validateFileNameRecursively(fullFilePath);
            }
301
            if (!result) {
302
303
                return Promise.reject(new Error(`file name in ${fullFilePath} is not valid!`));
            }
304
        } catch (error) {
305
306
307
            return Promise.reject(error);
        }
    }
308
    return Promise.resolve(result);
309
310
}

311
312
313
314
/**
 * get the version of current package
 */
async function getVersion(): Promise<string> {
chicm-ms's avatar
chicm-ms committed
315
    const deferred: Deferred<string> = new Deferred<string>();
316
    import(path.join(__dirname, '..', 'package.json')).then((pkg) => {
317
        deferred.resolve(pkg.version);
318
    }).catch((error) => {
319
320
321
        deferred.reject(error);
    });
    return deferred.promise;
322
}
323

324
325
326
/**
 * run command as ChildProcess
 */
327
function getTunerProc(command: string, stdio: StdioOptions, newCwd: string, newEnv: any): ChildProcess {
328
329
330
    let cmd: string = command;
    let arg: string[] = [];
    let newShell: boolean = true;
331
    if (process.platform === "win32") {
332
        cmd = command.split(" ", 1)[0];
333
        arg = command.substr(cmd.length + 1).split(" ");
334
335
336
337
338
339
340
341
342
343
344
345
346
347
        newShell = false;
    }
    const tunerProc: ChildProcess = spawn(cmd, arg, {
        stdio,
        cwd: newCwd,
        env: newEnv,
        shell: newShell
    });
    return tunerProc;
}

/**
 * judge whether the process is alive
 */
Yuge Zhang's avatar
Yuge Zhang committed
348
async function isAlive(pid: any): Promise<boolean> {
chicm-ms's avatar
chicm-ms committed
349
    const deferred: Deferred<boolean> = new Deferred<boolean>();
350
    let alive: boolean = false;
Yuge Zhang's avatar
Yuge Zhang committed
351
    if (process.platform === 'win32') {
352
353
354
355
356
357
358
        try {
            const str = cp.execSync(`powershell.exe Get-Process -Id ${pid} -ErrorAction SilentlyContinue`).toString();
            if (str) {
                alive = true;
            }
        }
        catch (error) {
chicm-ms's avatar
chicm-ms committed
359
            //ignore
360
361
        }
    }
Yuge Zhang's avatar
Yuge Zhang committed
362
    else {
363
364
365
366
367
368
369
370
371
372
373
374
        try {
            await cpp.exec(`kill -0 ${pid}`);
            alive = true;
        } catch (error) {
            //ignore
        }
    }
    deferred.resolve(alive);
    return deferred.promise;
}

/**
375
 * kill process
376
 */
Yuge Zhang's avatar
Yuge Zhang committed
377
async function killPid(pid: any): Promise<void> {
chicm-ms's avatar
chicm-ms committed
378
    const deferred: Deferred<void> = new Deferred<void>();
379
380
    try {
        if (process.platform === "win32") {
Yuge Zhang's avatar
Yuge Zhang committed
381
            await cpp.exec(`cmd.exe /c taskkill /PID ${pid} /F`);
382
        }
383
        else {
384
385
386
387
388
389
390
391
392
            await cpp.exec(`kill -9 ${pid}`);
        }
    } catch (error) {
        // pid does not exist, do nothing here
    }
    deferred.resolve();
    return deferred.promise;
}

393
function getNewLine(): string {
394
395
396
    if (process.platform === "win32") {
        return "\r\n";
    }
397
    else {
398
399
400
401
        return "\n";
    }
}

402
403
/**
 * Use '/' to join path instead of '\' for all kinds of platform
404
 * @param path
405
406
407
408
409
410
411
 */
function unixPathJoin(...paths: any[]): string {
    const dir: string = paths.filter((path: any) => path !== '').join('/');
    if (dir === '') return '.';
    return dir;
}

412
413
export {
    countFilesRecursively, validateFileNameRecursively, generateParamFileName, getMsgDispatcherCommand, getCheckpointDir,
414
    getLogDir, getExperimentRootDir, getJobCancelStatus, getDefaultDatabaseDir, getIPV4Address, unixPathJoin,
415
416
    mkDirP, mkDirPSync, delay, prepareUnitTest, parseArg, cleanupUnitTest, uniqueString, randomInt, randomSelect, getLogLevel, getVersion, getCmdPy, getTunerProc, isAlive, killPid, getNewLine
};