utils.ts 11 KB
Newer Older
liuzhe-lz's avatar
liuzhe-lz committed
1
2
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
Deshui Yu's avatar
Deshui Yu committed
3

4
import assert from 'assert';
Deshui Yu's avatar
Deshui Yu committed
5
import { randomBytes } from 'crypto';
6
7
import cpp from 'child-process-promise';
import cp from 'child_process';
8
import { ChildProcess, spawn, StdioOptions } from 'child_process';
9
10
11
12
13
import dgram from 'dgram';
import fs from 'fs';
import net from 'net';
import os from 'os';
import path from 'path';
liuzhe-lz's avatar
liuzhe-lz committed
14
import * as timersPromises from 'timers/promises';
Deshui Yu's avatar
Deshui Yu committed
15
16
17
18
import { Deferred } from 'ts-deferred';
import { Container } from 'typescript-ioc';

import { Database, DataStore } from './datastore';
19
20
import globals from './globals';
import { resetGlobals } from './globals/unittest';  // TODO: this file should not contain unittest helpers
21
import { ExperimentConfig, Manager } from './manager';
QuanluZhang's avatar
QuanluZhang committed
22
import { HyperParameters, TrainingService, TrialJobStatus } from './trainingService';
Deshui Yu's avatar
Deshui Yu committed
23

24
function getExperimentRootDir(): string {
25
    return globals.paths.experimentRoot;
Deshui Yu's avatar
Deshui Yu committed
26
27
}

28
function getLogDir(): string {
29
    return globals.paths.logDirectory;
Deshui Yu's avatar
Deshui Yu committed
30
31
}

32
function getLogLevel(): string {
33
    return globals.args.logLevel;
34
35
}

Deshui Yu's avatar
Deshui Yu committed
36
37
38
39
function getDefaultDatabaseDir(): string {
    return path.join(getExperimentRootDir(), 'db');
}

QuanluZhang's avatar
QuanluZhang committed
40
41
42
43
function getCheckpointDir(): string {
    return path.join(getExperimentRootDir(), 'checkpoint');
}

liuzhe-lz's avatar
liuzhe-lz committed
44
45
async function mkDirP(dirPath: string): Promise<void> {
    await fs.promises.mkdir(dirPath, { recursive: true });
Deshui Yu's avatar
Deshui Yu committed
46
47
48
}

function mkDirPSync(dirPath: string): void {
liuzhe-lz's avatar
liuzhe-lz committed
49
    fs.mkdirSync(dirPath, { recursive: true });
Deshui Yu's avatar
Deshui Yu committed
50
51
}

liuzhe-lz's avatar
liuzhe-lz committed
52
const delay = timersPromises.setTimeout;
Deshui Yu's avatar
Deshui Yu committed
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90

/**
 * Convert index to character
 * @param index index
 * @returns a mapping character
 */
function charMap(index: number): number {
    if (index < 26) {
        return index + 97;
    } else if (index < 52) {
        return index - 26 + 65;
    } else {
        return index - 52 + 48;
    }
}

/**
 * Generate a unique string by length
 * @param len length of string
 * @returns a unique string
 */
function uniqueString(len: number): string {
    if (len === 0) {
        return '';
    }
    const byteLength: number = Math.ceil((Math.log2(52) + Math.log2(62) * (len - 1)) / 8);
    let num: number = randomBytes(byteLength).reduce((a: number, b: number) => a * 256 + b, 0);
    const codes: number[] = [];
    codes.push(charMap(num % 52));
    num = Math.floor(num / 52);
    for (let i: number = 1; i < len; i++) {
        codes.push(charMap(num % 62));
        num = Math.floor(num / 62);
    }

    return String.fromCharCode(...codes);
}

91
92
93
94
function randomInt(max: number): number {
    return Math.floor(Math.random() * max);
}

95
96
97
98
99
function randomSelect<T>(a: T[]): T {
    assert(a !== undefined);

    return a[Math.floor(Math.random() * a.length)];
}
100

101
function getCmdPy(): string {
102
    let cmd = 'python3';
103
    if (process.platform === 'win32') {
104
105
106
107
108
        cmd = 'python';
    }
    return cmd;
}

109
/**
110
 * Generate command line to start automl algorithm(s),
QuanluZhang's avatar
QuanluZhang committed
111
 * either start advisor or start a process which runs tuner and assessor
112
 *
chicm-ms's avatar
chicm-ms committed
113
 * @param expParams: experiment startup parameters
114
115
 *
 */
116
function getMsgDispatcherCommand(expParams: ExperimentConfig): string {
chicm-ms's avatar
chicm-ms committed
117
118
119
    const clonedParams = Object.assign({}, expParams);
    delete clonedParams.searchSpace;
    return `${getCmdPy()} -m nni --exp_params ${Buffer.from(JSON.stringify(clonedParams)).toString('base64')}`;
120
121
}

122
123
124
125
/**
 * Generate parameter file name based on HyperParameters object
 * @param hyperParameters HyperParameters instance
 */
chicm-ms's avatar
chicm-ms committed
126
function generateParamFileName(hyperParameters: HyperParameters): string {
127
128
129
    assert(hyperParameters !== undefined);
    assert(hyperParameters.index >= 0);

chicm-ms's avatar
chicm-ms committed
130
    let paramFileName: string;
131
    if (hyperParameters.index == 0) {
132
133
134
135
136
137
138
        paramFileName = 'parameter.cfg';
    } else {
        paramFileName = `parameter_${hyperParameters.index}.cfg`
    }
    return paramFileName;
}

Deshui Yu's avatar
Deshui Yu committed
139
140
141
142
143
144
145
146
147
148
/**
 * Initialize a pseudo experiment environment for unit test.
 * Must be paired with `cleanupUnitTest()`.
 */
function prepareUnitTest(): void {
    Container.snapshot(Database);
    Container.snapshot(DataStore);
    Container.snapshot(TrainingService);
    Container.snapshot(Manager);

149
    resetGlobals();
Deshui Yu's avatar
Deshui Yu committed
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169

    const sqliteFile: string = path.join(getDefaultDatabaseDir(), 'nni.sqlite');
    try {
        fs.unlinkSync(sqliteFile);
    } catch (err) {
        // file not exists, good
    }
}

/**
 * Clean up unit test pseudo experiment.
 * Must be paired with `prepareUnitTest()`.
 */
function cleanupUnitTest(): void {
    Container.restore(Manager);
    Container.restore(TrainingService);
    Container.restore(DataStore);
    Container.restore(Database);
}

170
171
let cachedIpv4Address: string | null = null;

172
/**
173
 * Get IPv4 address of current machine.
174
 */
liuzhe-lz's avatar
liuzhe-lz committed
175
async function getIPV4Address(): Promise<string> {
176
177
    if (cachedIpv4Address !== null) {
        return cachedIpv4Address;
178
    }
179

180
181
182
183
    // creates "udp connection" to a non-exist target, and get local address of the connection.
    // since udp is connectionless, this does not send actual packets.
    const socket = dgram.createSocket('udp4');
    socket.connect(1, '192.0.2.0');
liuzhe-lz's avatar
liuzhe-lz committed
184
    for (let i = 0; i < 10; i++) {  // wait the system to initialize "connection"
liuzhe-lz's avatar
liuzhe-lz committed
185
186
187
188
189
190
191
192
        await timersPromises.setTimeout(1);
        try {
            cachedIpv4Address = socket.address().address;
            socket.close();
            return cachedIpv4Address;
        } catch (error) {
            /* retry */
        }
liuzhe-lz's avatar
liuzhe-lz committed
193
    }
liuzhe-lz's avatar
liuzhe-lz committed
194

liuzhe-lz's avatar
liuzhe-lz committed
195
    cachedIpv4Address = socket.address().address;  // if it still fails, throw the error
196
197
    socket.close();
    return cachedIpv4Address;
198
199
}

QuanluZhang's avatar
QuanluZhang committed
200
201
202
203
204
205
206
/**
 * Get the status of canceled jobs according to the hint isEarlyStopped
 */
function getJobCancelStatus(isEarlyStopped: boolean): TrialJobStatus {
    return isEarlyStopped ? 'EARLY_STOPPED' : 'USER_CANCELED';
}

207
208
209
210
/**
 * Utility method to calculate file numbers under a directory, recursively
 * @param directory directory name
 */
chicm-ms's avatar
chicm-ms committed
211
function countFilesRecursively(directory: string): Promise<number> {
212
    if (!fs.existsSync(directory)) {
213
214
215
216
217
        throw Error(`Direcotory ${directory} doesn't exist`);
    }

    const deferred: Deferred<number> = new Deferred<number>();

chicm-ms's avatar
chicm-ms committed
218
    let timeoutId: NodeJS.Timer
219
    const delayTimeout: Promise<number> = new Promise((_resolve: Function, reject: Function): void => {
220
221
222
223
224
225
226
        // Set timeout and reject the promise once reach timeout (5 seconds)
        timeoutId = setTimeout(() => {
            reject(new Error(`Timeout: path ${directory} has too many files`));
        }, 5000);
    });

    let fileCount: number = -1;
227
    let cmd: string;
228
    if (process.platform === "win32") {
229
230
        cmd = `powershell "Get-ChildItem -Path ${directory} -Recurse -File | Measure-Object | %{$_.Count}"`
    } else {
231
        cmd = `find ${directory} -type f | wc -l`;
232
233
    }
    cpp.exec(cmd).then((result) => {
234
        if (result.stdout && parseInt(result.stdout)) {
235
            fileCount = parseInt(result.stdout);
236
237
238
239
240
241
242
243
        }
        deferred.resolve(fileCount);
    });
    return Promise.race([deferred.promise, delayTimeout]).finally(() => {
        clearTimeout(timeoutId);
    });
}

244
245
246
247
/**
 * get the version of current package
 */
async function getVersion(): Promise<string> {
chicm-ms's avatar
chicm-ms committed
248
    const deferred: Deferred<string> = new Deferred<string>();
249
    import(path.join(__dirname, '..', 'package.json')).then((pkg) => {
250
        deferred.resolve(pkg.version);
251
252
    }).catch(() => {
        deferred.resolve('999.0.0-developing');
253
254
    });
    return deferred.promise;
255
}
256

257
258
259
/**
 * run command as ChildProcess
 */
J-shang's avatar
J-shang committed
260
function getTunerProc(command: string, stdio: StdioOptions, newCwd: string, newEnv: any, newShell: boolean = true, isDetached: boolean = false): ChildProcess {
261
262
    let cmd: string = command;
    let arg: string[] = [];
263
    if (process.platform === "win32") {
264
        cmd = command.split(" ", 1)[0];
265
        arg = command.substr(cmd.length + 1).split(" ");
266
        newShell = false;
267
        isDetached = true;
268
269
270
271
272
    }
    const tunerProc: ChildProcess = spawn(cmd, arg, {
        stdio,
        cwd: newCwd,
        env: newEnv,
273
274
        shell: newShell,
        detached: isDetached
275
276
277
278
279
280
281
    });
    return tunerProc;
}

/**
 * judge whether the process is alive
 */
Yuge Zhang's avatar
Yuge Zhang committed
282
async function isAlive(pid: any): Promise<boolean> {
chicm-ms's avatar
chicm-ms committed
283
    const deferred: Deferred<boolean> = new Deferred<boolean>();
284
    let alive: boolean = false;
Yuge Zhang's avatar
Yuge Zhang committed
285
    if (process.platform === 'win32') {
286
287
288
289
290
291
292
        try {
            const str = cp.execSync(`powershell.exe Get-Process -Id ${pid} -ErrorAction SilentlyContinue`).toString();
            if (str) {
                alive = true;
            }
        }
        catch (error) {
chicm-ms's avatar
chicm-ms committed
293
            //ignore
294
295
        }
    }
Yuge Zhang's avatar
Yuge Zhang committed
296
    else {
297
298
299
300
301
302
303
304
305
306
307
308
        try {
            await cpp.exec(`kill -0 ${pid}`);
            alive = true;
        } catch (error) {
            //ignore
        }
    }
    deferred.resolve(alive);
    return deferred.promise;
}

/**
309
 * kill process
310
 */
Yuge Zhang's avatar
Yuge Zhang committed
311
async function killPid(pid: any): Promise<void> {
chicm-ms's avatar
chicm-ms committed
312
    const deferred: Deferred<void> = new Deferred<void>();
313
314
    try {
        if (process.platform === "win32") {
Yuge Zhang's avatar
Yuge Zhang committed
315
            await cpp.exec(`cmd.exe /c taskkill /PID ${pid} /F`);
316
        }
317
        else {
318
319
320
321
322
323
324
325
326
            await cpp.exec(`kill -9 ${pid}`);
        }
    } catch (error) {
        // pid does not exist, do nothing here
    }
    deferred.resolve();
    return deferred.promise;
}

327
function getNewLine(): string {
328
329
330
    if (process.platform === "win32") {
        return "\r\n";
    }
331
    else {
332
333
334
335
        return "\n";
    }
}

336
337
/**
 * Use '/' to join path instead of '\' for all kinds of platform
338
 * @param path
339
340
341
342
343
344
345
 */
function unixPathJoin(...paths: any[]): string {
    const dir: string = paths.filter((path: any) => path !== '').join('/');
    if (dir === '') return '.';
    return dir;
}

J-shang's avatar
J-shang committed
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
async function isPortOpen(host: string, port: number): Promise<boolean> {
    return new Promise<boolean>((resolve, reject) => {
        try{
            const stream = net.createConnection(port, host);
            const id = setTimeout(() => {
                stream.destroy();
                resolve(false);
            }, 1000);

            stream.on('connect', () => {
                clearTimeout(id);
                stream.destroy();
                resolve(true);
            });

            stream.on('error', () => {
                clearTimeout(id);
                stream.destroy();
                resolve(false);
            });
        } catch (error) {
            reject(error);
        }
    });
}

async function getFreePort(host: string, start: number, end: number): Promise<number> {
    if (start > end) {
        throw new Error(`no more free port`);
    }
    if (await isPortOpen(host, start)) {
        return await getFreePort(host, start + 1, end);
    } else {
        return start;
    }
}

383
384
385
386
387
export function importModule(modulePath: string): any {
    module.paths.unshift(path.dirname(modulePath));
    return require(path.basename(modulePath));
}

388
export {
389
390
    countFilesRecursively, generateParamFileName, getMsgDispatcherCommand, getCheckpointDir,
    getLogDir, getExperimentRootDir, getJobCancelStatus, getDefaultDatabaseDir, getIPV4Address, unixPathJoin, getFreePort, isPortOpen,
liuzhe-lz's avatar
liuzhe-lz committed
391
    mkDirP, mkDirPSync, delay, prepareUnitTest, cleanupUnitTest, uniqueString, randomInt, randomSelect, getLogLevel, getVersion, getCmdPy, getTunerProc, isAlive, killPid, getNewLine
392
};