gpuScheduler.ts 3.71 KB
Newer Older
liuzhe-lz's avatar
liuzhe-lz committed
1
2
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
Deshui Yu's avatar
Deshui Yu committed
3
4
5

'use strict';

6
7
import * as cpp from 'child-process-promise';
import * as fs from 'fs';
8
9
10
11
import * as os from 'os';
import * as path from 'path';
import { getLogger, Logger } from '../../common/log';
import { delay } from '../../common/utils';
12
import { GPUInfo, GPUSummary } from '../common/gpuData';
13
import { execKill, execMkdir, execRemove, execTail, runGpuMetricsCollector } from '../common/util';
Deshui Yu's avatar
Deshui Yu committed
14
15

/**
16
 * GPUScheduler for local training service
Deshui Yu's avatar
Deshui Yu committed
17
18
19
20
21
 */
class GPUScheduler {

    private gpuSummary!: GPUSummary;
    private stopping: boolean;
22
23
    private readonly log: Logger;
    private readonly gpuMetricCollectorScriptFolder: string;
Deshui Yu's avatar
Deshui Yu committed
24
25
26

    constructor() {
        this.stopping = false;
27
        this.log = getLogger();
28
        this.gpuMetricCollectorScriptFolder = `${os.tmpdir()}/${os.userInfo().username}/nni/script`;
Deshui Yu's avatar
Deshui Yu committed
29
30
31
    }

    public async run(): Promise<void> {
32
        await this.runGpuMetricsCollectorScript();
Deshui Yu's avatar
Deshui Yu committed
33
34
        while (!this.stopping) {
            try {
35
                await this.updateGPUSummary();
Deshui Yu's avatar
Deshui Yu committed
36
            } catch (error) {
37
                this.log.error('Read GPU summary failed with error: ', error);
Deshui Yu's avatar
Deshui Yu committed
38
            }
39
40
41
            if (this.gpuSummary !== undefined && this.gpuSummary.gpuCount === 0) {
                throw new Error('GPU not available. Please check your CUDA configuration');
            }
Deshui Yu's avatar
Deshui Yu committed
42
43
44
45
            await delay(5000);
        }
    }

46
    public getAvailableGPUIndices(useActiveGpu: boolean, occupiedGpuIndexNumMap: Map<number, number>): number[] {
Deshui Yu's avatar
Deshui Yu committed
47
        if (this.gpuSummary !== undefined) {
48
            if (process.platform === 'win32' || useActiveGpu) {
demianzhang's avatar
demianzhang committed
49
                return this.gpuSummary.gpuInfos.map((info: GPUInfo) => info.index);
50
51
52
53
54
            } else {
                return this.gpuSummary.gpuInfos.filter((info: GPUInfo) =>
                         occupiedGpuIndexNumMap.get(info.index) === undefined && info.activeProcessNum === 0 ||
                         occupiedGpuIndexNumMap.get(info.index) !== undefined)
                       .map((info: GPUInfo) => info.index);
demianzhang's avatar
demianzhang committed
55
            }
Deshui Yu's avatar
Deshui Yu committed
56
57
58
59
60
        }

        return [];
    }

61
62
63
64
65
66
67
68
69
    public getSystemGpuCount(): number {
        if (this.gpuSummary !== undefined) {
            return this.gpuSummary.gpuCount;
        }

        return 0;
    }

    public async stop(): Promise<void> {
Deshui Yu's avatar
Deshui Yu committed
70
        this.stopping = true;
71
72
        try {
            const pid: string = await fs.promises.readFile(path.join(this.gpuMetricCollectorScriptFolder, 'pid'), 'utf8');
73
74
            await execKill(pid);
            await execRemove(this.gpuMetricCollectorScriptFolder);
75
        } catch (error) {
76
77
            this.log.error(`GPU scheduler error: ${error}`);
        }
Deshui Yu's avatar
Deshui Yu committed
78
79
    }

80
81
82
83
84
    /**
     * Generate gpu metric collector shell script in local machine,
     * used to run in remote machine, and will be deleted after uploaded from local.
     */
    private async runGpuMetricsCollectorScript(): Promise<void> {
85
        await execMkdir(this.gpuMetricCollectorScriptFolder, true);
86
        runGpuMetricsCollector(this.gpuMetricCollectorScriptFolder);
87
88
    }

89
    private async updateGPUSummary(): Promise<void> {
90
        const gpuMetricPath: string = path.join(this.gpuMetricCollectorScriptFolder, 'gpu_metrics');
91
92
        if (fs.existsSync(gpuMetricPath)) {
            const cmdresult: cpp.childProcessPromise.Result = await execTail(gpuMetricPath);
93
            if (cmdresult !== undefined && cmdresult.stdout !== undefined) {
94
95
96
97
                this.gpuSummary = <GPUSummary>JSON.parse(cmdresult.stdout);
            } else {
                this.log.error('Could not get gpu metrics information!');
            }
98
99
        } else {
            this.log.warning('gpu_metrics file does not exist!');
100
        }
Deshui Yu's avatar
Deshui Yu committed
101
102
103
104
    }
}

export { GPUScheduler };