gpuScheduler.ts 4.1 KB
Newer Older
Deshui Yu's avatar
Deshui Yu committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
/**
 * Copyright (c) Microsoft Corporation
 * All rights reserved.
 *
 * MIT License
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
 * documentation files (the "Software"), to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
 * to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
 * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

'use strict';

import { delay } from '../../common/utils';
import { GPUInfo, GPUSummary } from '../common/gpuData';
24
import { getLogger, Logger } from '../../common/log';
25
26
27
28
29
30
31
import * as cp from 'child_process';
import * as cpp from 'child-process-promise';
import * as path from 'path';
import * as os from 'os';
import * as fs from 'fs';
import { String } from 'typescript-string-operations';
import { GPU_INFO_COLLECTOR_FORMAT } from '../common/gpuData'
Deshui Yu's avatar
Deshui Yu committed
32
33
34
35
36
37
38
39

/**
 * GPUScheduler
 */
class GPUScheduler {

    private gpuSummary!: GPUSummary;
    private stopping: boolean;
40
    private log: Logger;
41
    private gpuMetricCollectorScriptFolder: string;
Deshui Yu's avatar
Deshui Yu committed
42
43
44

    constructor() {
        this.stopping = false;
45
        this.log = getLogger();
46
        this.gpuMetricCollectorScriptFolder = `${os.tmpdir()}/nni/script`;
Deshui Yu's avatar
Deshui Yu committed
47
48
49
    }

    public async run(): Promise<void> {
50
        await this.runGpuMetricsCollectorScript();
Deshui Yu's avatar
Deshui Yu committed
51
52
        while (!this.stopping) {
            try {
53
                await this.updateGPUSummary();
Deshui Yu's avatar
Deshui Yu committed
54
            } catch (error) {
55
                this.log.error('Read GPU summary failed with error: ', error);
Deshui Yu's avatar
Deshui Yu committed
56
57
58
59
60
            }
            await delay(5000);
        }
    }

61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
    /**
     * Generate gpu metric collector shell script in local machine, 
     * used to run in remote machine, and will be deleted after uploaded from local. 
     */
    private async runGpuMetricsCollectorScript(): Promise<void> {
        await cpp.exec(`mkdir -p ${this.gpuMetricCollectorScriptFolder}`);
        //generate gpu_metrics_collector.sh
        let gpuMetricsCollectorScriptPath: string = path.join(this.gpuMetricCollectorScriptFolder, 'gpu_metrics_collector.sh');
        const gpuMetricsCollectorScriptContent: string = String.Format(
            GPU_INFO_COLLECTOR_FORMAT,
            this.gpuMetricCollectorScriptFolder,
            path.join(this.gpuMetricCollectorScriptFolder, 'pid'),
        );
        await fs.promises.writeFile(gpuMetricsCollectorScriptPath, gpuMetricsCollectorScriptContent, { encoding: 'utf8' });
        cp.exec(`bash ${gpuMetricsCollectorScriptPath}`);
    }

Deshui Yu's avatar
Deshui Yu committed
78
79
80
81
82
83
84
85
    public getAvailableGPUIndices(): number[] {
        if (this.gpuSummary !== undefined) {
            return this.gpuSummary.gpuInfos.filter((info: GPUInfo) => info.activeProcessNum === 0).map((info: GPUInfo) => info.index);
        }

        return [];
    }

86
    public async stop() {
Deshui Yu's avatar
Deshui Yu committed
87
        this.stopping = true;
88
89
90
        const pid: string = await fs.promises.readFile(path.join(this.gpuMetricCollectorScriptFolder, 'pid'), 'utf8');
        await cpp.exec(`pkill -P ${pid}`);
        await cpp.exec(`rm -rf ${this.gpuMetricCollectorScriptFolder}`);
Deshui Yu's avatar
Deshui Yu committed
91
92
    }

93
94
95
96
    private async updateGPUSummary() {
        const cmdresult = await cpp.exec(`tail -n 1 ${path.join(this.gpuMetricCollectorScriptFolder, 'gpu_metrics')}`);
        if(cmdresult && cmdresult.stdout) {
            this.gpuSummary = <GPUSummary>JSON.parse(cmdresult.stdout);
97
        } else {
98
            this.log.error('Could not get gpu metrics information!');
99
        }
Deshui Yu's avatar
Deshui Yu committed
100
101
102
103
    }
}

export { GPUScheduler };