remoteMachineTrainingService.ts 31.2 KB
Newer Older
liuzhe-lz's avatar
liuzhe-lz committed
1
2
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
Deshui Yu's avatar
Deshui Yu committed
3
4
5

'use strict';

6
import * as assert from 'assert';
Deshui Yu's avatar
Deshui Yu committed
7
8
9
10
11
import * as cpp from 'child-process-promise';
import { EventEmitter } from 'events';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
12
import { Client, ConnectConfig } from 'ssh2';
Deshui Yu's avatar
Deshui Yu committed
13
14
15
import { Deferred } from 'ts-deferred';
import { String } from 'typescript-string-operations';
import * as component from '../../common/component';
SparkSnail's avatar
SparkSnail committed
16
import { NNIError, NNIErrorNames } from '../../common/errors';
17
import { getExperimentId } from '../../common/experimentStartupInfo';
Deshui Yu's avatar
Deshui Yu committed
18
19
20
import { getLogger, Logger } from '../../common/log';
import { ObservableTimer } from '../../common/observableTimer';
import {
21
    HyperParameters, NNIManagerIpConfig, TrainingService, TrialJobApplicationForm,
22
    TrialJobDetail, TrialJobMetric
Deshui Yu's avatar
Deshui Yu committed
23
} from '../../common/trainingService';
24
25
26
27
28
import {
    delay, generateParamFileName, getExperimentRootDir, getIPV4Address, getJobCancelStatus, getRemoteTmpDir,
    getVersion, uniqueString, unixPathJoin
} from '../../common/utils';
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData';
29
import { GPUSummary } from '../common/gpuData';
30
31
import { TrialConfig } from '../common/trialConfig';
import { TrialConfigMetadataKey } from '../common/trialConfigMetadataKey';
32
import { execCopydir, execMkdir, execRemove, validateCodeDir, getGpuMetricsCollectorBashScriptContent } from '../common/util';
Deshui Yu's avatar
Deshui Yu committed
33
34
import { GPUScheduler } from './gpuScheduler';
import {
35
36
37
    HOST_JOB_SHELL_FORMAT, RemoteCommandResult, REMOTEMACHINE_TRIAL_COMMAND_FORMAT, RemoteMachineMeta,
    RemoteMachineScheduleInfo, RemoteMachineScheduleResult, RemoteMachineTrialJobDetail,
    ScheduleResultType, SSHClient, SSHClientManager
Deshui Yu's avatar
Deshui Yu committed
38
} from './remoteMachineData';
SparkSnail's avatar
SparkSnail committed
39
import { RemoteMachineJobRestServer } from './remoteMachineJobRestServer';
40
import { SSHClientUtility } from './sshClientUtility';
Deshui Yu's avatar
Deshui Yu committed
41
42
43
44

/**
 * Training Service implementation for Remote Machine (Linux)
 */
SparkSnail's avatar
SparkSnail committed
45
@component.Singleton
Deshui Yu's avatar
Deshui Yu committed
46
class RemoteMachineTrainingService implements TrainingService {
47
48
49
50
51
52
    private readonly machineSSHClientMap: Map<RemoteMachineMeta, SSHClientManager>; //machine ssh client map
    private readonly trialSSHClientMap: Map<string, Client>; //trial ssh client map
    private readonly trialJobsMap: Map<string, RemoteMachineTrialJobDetail>;
    private readonly MAX_TRIAL_NUMBER_PER_SSHCONNECTION: number = 5; // every ssh client has a max trial concurrency number
    private readonly expRootDir: string;
    private readonly remoteExpRootDir: string;
53
    private trialConfig: TrialConfig | undefined;
54
    private gpuScheduler?: GPUScheduler;
55
56
    private readonly jobQueue: string[];
    private readonly timer: ObservableTimer;
Deshui Yu's avatar
Deshui Yu committed
57
    private stopping: boolean = false;
58
59
    private readonly metricsEmitter: EventEmitter;
    private readonly log: Logger;
60
    private isMultiPhase: boolean = false;
61
    private trialSequenceId: number;
SparkSnail's avatar
SparkSnail committed
62
    private remoteRestServerPort?: number;
63
    private readonly remoteOS: string;
SparkSnail's avatar
SparkSnail committed
64
    private nniManagerIpConfig?: NNIManagerIpConfig;
65
    private versionCheck: boolean = true;
SparkSnail's avatar
SparkSnail committed
66
    private logCollection: string;
Deshui Yu's avatar
Deshui Yu committed
67
68

    constructor(@component.Inject timer: ObservableTimer) {
69
        this.remoteOS = 'linux';
Deshui Yu's avatar
Deshui Yu committed
70
71
        this.metricsEmitter = new EventEmitter();
        this.trialJobsMap = new Map<string, RemoteMachineTrialJobDetail>();
SparkSnail's avatar
SparkSnail committed
72
73
        this.trialSSHClientMap = new Map<string, Client>();
        this.machineSSHClientMap = new Map<RemoteMachineMeta, SSHClientManager>();
Deshui Yu's avatar
Deshui Yu committed
74
75
        this.jobQueue = [];
        this.expRootDir = getExperimentRootDir();
76
        this.remoteExpRootDir = this.getRemoteExperimentRootDir();
Deshui Yu's avatar
Deshui Yu committed
77
78
        this.timer = timer;
        this.log = getLogger();
79
        this.trialSequenceId = -1;
SparkSnail's avatar
SparkSnail committed
80
        this.logCollection = 'none';
chicm-ms's avatar
chicm-ms committed
81
        this.log.info('Construct remote machine training service.');
Deshui Yu's avatar
Deshui Yu committed
82
83
84
85
86
87
    }

    /**
     * Loop to launch trial jobs and collect trial metrics
     */
    public async run(): Promise<void> {
SparkSnail's avatar
SparkSnail committed
88
89
        const restServer: RemoteMachineJobRestServer = component.get(RemoteMachineJobRestServer);
        await restServer.start();
90
        restServer.setEnableVersionCheck = this.versionCheck;
chicm-ms's avatar
chicm-ms committed
91
        this.log.info('Run remote machine training service.');
Deshui Yu's avatar
Deshui Yu committed
92
93
        while (!this.stopping) {
            while (this.jobQueue.length > 0) {
SparkSnail's avatar
SparkSnail committed
94
                this.updateGpuReservation();
Deshui Yu's avatar
Deshui Yu committed
95
96
97
98
99
100
                const trialJobId: string = this.jobQueue[0];
                const prepareResult : boolean = await this.prepareTrialJob(trialJobId);
                if (prepareResult) {
                    // Remove trial job with trialJobId from job queue
                    this.jobQueue.shift();
                } else {
101
                    // Break the while loop since no GPU resource is available right now,
Deshui Yu's avatar
Deshui Yu committed
102
103
104
                    // Wait to schedule job in next time iteration
                    break;
                }
105
            }
106
            if (restServer.getErrorMessage !== undefined) {
107
108
109
                throw new Error(restServer.getErrorMessage);
                this.stopping = true;
            }
Deshui Yu's avatar
Deshui Yu committed
110
111
            await delay(3000);
        }
chicm-ms's avatar
chicm-ms committed
112
        this.log.info('Remote machine training service exit.');
Deshui Yu's avatar
Deshui Yu committed
113
    }
114

SparkSnail's avatar
SparkSnail committed
115
116
    /**
     * give trial a ssh connection
117
     * @param trial remote machine trial job detail
SparkSnail's avatar
SparkSnail committed
118
119
120
     */
    public async allocateSSHClientForTrial(trial: RemoteMachineTrialJobDetail): Promise<void> {
        const deferred: Deferred<void> = new Deferred<void>();
121
        if (trial.rmMeta === undefined) {
SparkSnail's avatar
SparkSnail committed
122
123
            throw new Error(`rmMeta not set in trial ${trial.id}`);
        }
124
125
        const sshClientManager: SSHClientManager | undefined = this.machineSSHClientMap.get(trial.rmMeta);
        if (sshClientManager === undefined) {
SparkSnail's avatar
SparkSnail committed
126
127
            throw new Error(`remoteSSHClient not initialized`);
        }
128
        const sshClient: Client = await sshClientManager.getAvailableSSHClient();
SparkSnail's avatar
SparkSnail committed
129
130
        this.trialSSHClientMap.set(trial.id, sshClient);
        deferred.resolve();
131

SparkSnail's avatar
SparkSnail committed
132
133
        return deferred.promise;
    }
134

SparkSnail's avatar
SparkSnail committed
135
136
    /**
     * If a trial is finished, release the connection resource
137
     * @param trial remote machine trial job detail
SparkSnail's avatar
SparkSnail committed
138
139
     */
    public releaseTrialSSHClient(trial: RemoteMachineTrialJobDetail): void {
140
        if (trial.rmMeta === undefined) {
SparkSnail's avatar
SparkSnail committed
141
142
            throw new Error(`rmMeta not set in trial ${trial.id}`);
        }
143
144
        const sshClientManager: SSHClientManager | undefined = this.machineSSHClientMap.get(trial.rmMeta);
        if (sshClientManager === undefined) {
SparkSnail's avatar
SparkSnail committed
145
146
147
148
            throw new Error(`sshClientManager not initialized`);
        }
        sshClientManager.releaseConnection(this.trialSSHClientMap.get(trial.id));
    }
Deshui Yu's avatar
Deshui Yu committed
149
150
151
152

    /**
     * List submitted trial jobs
     */
153
    public async listTrialJobs(): Promise<TrialJobDetail[]> {
Deshui Yu's avatar
Deshui Yu committed
154
155
156
        const jobs: TrialJobDetail[] = [];
        const deferred: Deferred<TrialJobDetail[]> = new Deferred<TrialJobDetail[]>();

157
        for (const [key, value] of this.trialJobsMap) {
158
            jobs.push(await this.getTrialJob(key));
159
        }
Deshui Yu's avatar
Deshui Yu committed
160
161
162
163
164
165
166
167
168
169
170
        deferred.resolve(jobs);

        return deferred.promise;
    }

    /**
     * Get trial job detail information
     * @param trialJobId ID of trial job
     */
    public async getTrialJob(trialJobId: string): Promise<TrialJobDetail> {
        const trialJob: RemoteMachineTrialJobDetail | undefined = this.trialJobsMap.get(trialJobId);
171
        if (trialJob === undefined) {
Deshui Yu's avatar
Deshui Yu committed
172
173
174
175
176
177
178
179
            throw new NNIError(NNIErrorNames.NOT_FOUND, `trial job id ${trialJobId} not found`);
        }
        //TO DO: add another job status, and design new job status change logic
        if (trialJob.status === 'RUNNING' || trialJob.status === 'UNKNOWN') {
            // Get ssh client where the job is running
            if (trialJob.rmMeta === undefined) {
                throw new Error(`rmMeta not set for submitted job ${trialJobId}`);
            }
SparkSnail's avatar
SparkSnail committed
180
            const sshClient: Client | undefined  = this.trialSSHClientMap.get(trialJob.id);
181
            if (sshClient === undefined) {
Deshui Yu's avatar
Deshui Yu committed
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
                throw new Error(`Invalid job id: ${trialJobId}, cannot find ssh client`);
            }

            return this.updateTrialJobStatus(trialJob, sshClient);
        } else {
            return trialJob;
        }
    }

    /**
     * Add job metrics listener
     * @param listener callback listener
     */
    public addTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void {
        this.metricsEmitter.on('metric', listener);
    }

    /**
     * Remove job metrics listener
     * @param listener callback listener
     */
    public removeTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void {
        this.metricsEmitter.off('metric', listener);
    }

    /**
     * Submit trial job
     * @param form trial job description form
     */
211
    // tslint:disable-next-line:informative-docs
212
    public async submitTrialJob(form: TrialJobApplicationForm): Promise<TrialJobDetail> {
213
        if (this.trialConfig === undefined) {
Deshui Yu's avatar
Deshui Yu committed
214
215
216
            throw new Error('trial config is not initialized');
        }

217
218
219
        // Generate trial job id(random)
        const trialJobId: string = uniqueString(5);
        const trialWorkingFolder: string = unixPathJoin(this.remoteExpRootDir, 'trials', trialJobId);
Deshui Yu's avatar
Deshui Yu committed
220

221
222
223
224
225
226
227
228
229
        const trialJobDetail: RemoteMachineTrialJobDetail = new RemoteMachineTrialJobDetail(
            trialJobId,
            'WAITING',
            Date.now(),
            trialWorkingFolder,
            form
        );
        this.jobQueue.push(trialJobId);
        this.trialJobsMap.set(trialJobId, trialJobDetail);
230

231
        return Promise.resolve(trialJobDetail);
Deshui Yu's avatar
Deshui Yu committed
232
233
    }

234
235
236
237
238
    /**
     * Update trial job for multi-phase
     * @param trialJobId trial job id
     * @param form job application form
     */
239
    public async updateTrialJob(trialJobId: string, form: TrialJobApplicationForm): Promise<TrialJobDetail> {
chicm-ms's avatar
chicm-ms committed
240
241
242
243
        const trialJobDetail: undefined | TrialJobDetail = this.trialJobsMap.get(trialJobId);
        if (trialJobDetail === undefined) {
            throw new Error(`updateTrialJob failed: ${trialJobId} not found`);
        }
244
245
246
        const rmMeta: RemoteMachineMeta | undefined = (<RemoteMachineTrialJobDetail>trialJobDetail).rmMeta;
        if (rmMeta !== undefined) {
            await this.writeParameterFile(trialJobId, form.hyperParameters, rmMeta);
chicm-ms's avatar
chicm-ms committed
247
        } else {
248
            throw new Error(`updateTrialJob failed: ${trialJobId} rmMeta not found`);
chicm-ms's avatar
chicm-ms committed
249
250
251
        }

        return trialJobDetail;
252
    }
253

254
255
256
257
    /**
     * Is multiphase job supported in current training service
     */
    public get isMultiPhaseJobSupported(): boolean {
258
        return true;
259
260
    }

Deshui Yu's avatar
Deshui Yu committed
261
262
263
264
    /**
     * Cancel trial job
     * @param trialJobId ID of trial job
     */
265
    // tslint:disable:informative-docs no-unsafe-any
QuanluZhang's avatar
QuanluZhang committed
266
    public async cancelTrialJob(trialJobId: string, isEarlyStopped: boolean = false): Promise<void> {
Deshui Yu's avatar
Deshui Yu committed
267
268
        const deferred: Deferred<void> = new Deferred<void>();
        const trialJob: RemoteMachineTrialJobDetail | undefined = this.trialJobsMap.get(trialJobId);
269
        if (trialJob === undefined) {
Deshui Yu's avatar
Deshui Yu committed
270
271
272
273
274
275
            deferred.reject();
            throw new Error(`trial job id ${trialJobId} not found`);
        }

        // Remove the job with trialJobId from job queue
        const index : number = this.jobQueue.indexOf(trialJobId);
276
        if (index >= 0) {
Deshui Yu's avatar
Deshui Yu committed
277
278
279
280
281
282
            this.jobQueue.splice(index, 1);
        }

        // Get ssh client where the job is running
        if (trialJob.rmMeta !== undefined) {
            // If the trial job is already scheduled, check its status and kill the trial process in remote machine
SparkSnail's avatar
SparkSnail committed
283
            const sshClient: Client | undefined = this.trialSSHClientMap.get(trialJob.id);
284
            if (sshClient === undefined) {
Deshui Yu's avatar
Deshui Yu committed
285
286
287
288
289
290
                deferred.reject();
                throw new Error(`Invalid job id ${trialJobId}, cannot find ssh client`);
            }

            const jobpidPath: string = this.getJobPidPath(trialJob.id);
            try {
291
292
                // Mark the toEarlyStop tag here
                trialJob.isEarlyStopped = isEarlyStopped;
Deshui Yu's avatar
Deshui Yu committed
293
                await SSHClientUtility.remoteExeCommand(`pkill -P \`cat ${jobpidPath}\``, sshClient);
SparkSnail's avatar
SparkSnail committed
294
                this.releaseTrialSSHClient(trialJob);
Deshui Yu's avatar
Deshui Yu committed
295
296
297
298
299
300
            } catch (error) {
                // Not handle the error since pkill failed will not impact trial job's current status
                this.log.error(`remoteTrainingService.cancelTrialJob: ${error.message}`);
            }
        } else {
            // Job is not scheduled yet, set status to 'USER_CANCELLED' directly
QuanluZhang's avatar
QuanluZhang committed
301
302
            assert(isEarlyStopped === false, 'isEarlyStopped is not supposed to be true here.');
            trialJob.status = getJobCancelStatus(isEarlyStopped);
Deshui Yu's avatar
Deshui Yu committed
303
304
305
306
307
308
309
310
311
312
313
314
        }
    }

    /**
     * Set culster metadata
     * @param key metadata key
     * //1. MACHINE_LIST -- create ssh client connect of machine list
     * //2. TRIAL_CONFIG -- trial configuration
     * @param value metadata value
     */
    public async setClusterMetadata(key: string, value: string): Promise<void> {
        switch (key) {
SparkSnail's avatar
SparkSnail committed
315
316
317
            case TrialConfigMetadataKey.NNI_MANAGER_IP:
                this.nniManagerIpConfig = <NNIManagerIpConfig>JSON.parse(value);
                break;
318
            case TrialConfigMetadataKey.MACHINE_LIST:
Deshui Yu's avatar
Deshui Yu committed
319
                await this.setupConnections(value);
320
                this.gpuScheduler = new GPUScheduler(this.machineSSHClientMap);
Deshui Yu's avatar
Deshui Yu committed
321
                break;
322
323
            case TrialConfigMetadataKey.TRIAL_CONFIG:
                const remoteMachineTrailConfig: TrialConfig = <TrialConfig>JSON.parse(value);
Deshui Yu's avatar
Deshui Yu committed
324
                // Parse trial config failed, throw Error
325
                if (remoteMachineTrailConfig === undefined) {
Deshui Yu's avatar
Deshui Yu committed
326
327
328
                    throw new Error('trial config parsed failed');
                }
                // codeDir is not a valid directory, throw Error
329
330
331
                // tslint:disable-next-line:non-literal-fs-path
                if (!fs.lstatSync(remoteMachineTrailConfig.codeDir)
                  .isDirectory()) {
Deshui Yu's avatar
Deshui Yu committed
332
333
                    throw new Error(`codeDir ${remoteMachineTrailConfig.codeDir} is not a directory`);
                }
334
335
336
337

                // Validate to make sure codeDir doesn't have too many files
                try {
                    await validateCodeDir(remoteMachineTrailConfig.codeDir);
338
                } catch (error) {
339
                    this.log.error(error);
340

341
                    return Promise.reject(new Error(error));
342
343
                }

Deshui Yu's avatar
Deshui Yu committed
344
345
                this.trialConfig = remoteMachineTrailConfig;
                break;
346
347
348
            case TrialConfigMetadataKey.MULTI_PHASE:
                this.isMultiPhase = (value === 'true' || value === 'True');
                break;
349
350
351
            case TrialConfigMetadataKey.VERSION_CHECK:
                this.versionCheck = (value === 'true' || value === 'True');
                break;
SparkSnail's avatar
SparkSnail committed
352
353
354
            case TrialConfigMetadataKey.LOG_COLLECTION:
                this.logCollection = value;
                break;
Deshui Yu's avatar
Deshui Yu committed
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
            default:
                //Reject for unknown keys
                throw new Error(`Uknown key: ${key}`);
        }
    }

    /**
     * Get culster metadata
     * @param key metadata key
     */
    public getClusterMetadata(key: string): Promise<string> {
        const deferred: Deferred<string> = new Deferred<string>();

        return deferred.promise;
    }
370

SparkSnail's avatar
SparkSnail committed
371
    /**
372
     * cleanup() has a time out of 10s to clean remote connections
SparkSnail's avatar
SparkSnail committed
373
374
     */
    public async cleanUp(): Promise<void> {
chicm-ms's avatar
chicm-ms committed
375
        this.log.info('Stopping remote machine training service...');
Deshui Yu's avatar
Deshui Yu committed
376
        this.stopping = true;
SparkSnail's avatar
SparkSnail committed
377
378
        await Promise.race([delay(10000), this.cleanupConnections()]);
    }
379

380
381
382
383
    /**
     * remove gpu reversion when job is not running
     */
    private updateGpuReservation(): void {
384
385
386
387
388
        if (this.gpuScheduler) {
            for (const [key, value] of this.trialJobsMap) {
                if (!['WAITING', 'RUNNING'].includes(value.status)) {
                    this.gpuScheduler.removeGpuReservation(key, this.trialJobsMap);
                }
389
390
391
392
            }
        }
    }

SparkSnail's avatar
SparkSnail committed
393
394
395
396
    /**
     * stop gpu_metric_collector process in remote machine and remove unused scripts
     */
    private async cleanupConnections(): Promise<void> {
397
        try {
SparkSnail's avatar
SparkSnail committed
398
            for (const [rmMeta, sshClientManager] of this.machineSSHClientMap.entries()) {
399
400
401
                const jobpidPath: string = unixPathJoin(this.getRemoteScriptsPath(rmMeta.username), 'pid');
                const client: Client | undefined = sshClientManager.getFirstSSHClient();
                if (client !== undefined) {
SparkSnail's avatar
SparkSnail committed
402
403
404
405
                    await SSHClientUtility.remoteExeCommand(`pkill -P \`cat ${jobpidPath}\``, client);
                    await SSHClientUtility.remoteExeCommand(`rm -rf ${this.getRemoteScriptsPath(rmMeta.username)}`, client);
                }
                sshClientManager.closeAllSSHClient();
SparkSnail's avatar
SparkSnail committed
406
            }
407
        } catch (error) {
SparkSnail's avatar
SparkSnail committed
408
409
410
            //ignore error, this function is called to cleanup remote connections when experiment is stopping
            this.log.error(`Cleanup connection exception, error is ${error.message}`);
        }
Deshui Yu's avatar
Deshui Yu committed
411
412

        return Promise.resolve();
413
414
    }

Deshui Yu's avatar
Deshui Yu committed
415
    private async setupConnections(machineList: string): Promise<void> {
chicm-ms's avatar
chicm-ms committed
416
        this.log.debug(`Connecting to remote machines: ${machineList}`);
Deshui Yu's avatar
Deshui Yu committed
417
418
419
420
        const deferred: Deferred<void> = new Deferred<void>();
        //TO DO: verify if value's format is wrong, and json parse failed, how to handle error
        const rmMetaList: RemoteMachineMeta[] = <RemoteMachineMeta[]>JSON.parse(machineList);
        let connectedRMNum: number = 0;
SparkSnail's avatar
SparkSnail committed
421

SparkSnail's avatar
SparkSnail committed
422
        rmMetaList.forEach(async (rmMeta: RemoteMachineMeta) => {
423
            rmMeta.occupiedGpuIndexMap = new Map<number, number>();
424
425
            const sshClientManager: SSHClientManager = new SSHClientManager([], this.MAX_TRIAL_NUMBER_PER_SSHCONNECTION, rmMeta);
            const sshClient: Client = await sshClientManager.getAvailableSSHClient();
SparkSnail's avatar
SparkSnail committed
426
427
428
429
            this.machineSSHClientMap.set(rmMeta, sshClientManager);
            await this.initRemoteMachineOnConnected(rmMeta, sshClient);
            if (++connectedRMNum === rmMetaList.length) {
                deferred.resolve();
430
            }
Deshui Yu's avatar
Deshui Yu committed
431
        });
432

Deshui Yu's avatar
Deshui Yu committed
433
434
435
436
437
        return deferred.promise;
    }

    private async initRemoteMachineOnConnected(rmMeta: RemoteMachineMeta, conn: Client): Promise<void> {
        // Create root working directory after ssh connection is ready
438
        const nniRootDir: string = unixPathJoin(getRemoteTmpDir(this.remoteOS), 'nni');
Deshui Yu's avatar
Deshui Yu committed
439
        await SSHClientUtility.remoteExeCommand(`mkdir -p ${this.remoteExpRootDir}`, conn);
440

441
442
        // the directory to store temp scripts in remote machine
        const remoteGpuScriptCollectorDir: string = this.getRemoteScriptsPath(rmMeta.username);
443
        await SSHClientUtility.remoteExeCommand(`(umask 0 ; mkdir -p ${remoteGpuScriptCollectorDir})`, conn);
Deshui Yu's avatar
Deshui Yu committed
444
        await SSHClientUtility.remoteExeCommand(`chmod 777 ${nniRootDir} ${nniRootDir}/* ${nniRootDir}/scripts/*`, conn);
445

Deshui Yu's avatar
Deshui Yu committed
446
        //Begin to execute gpu_metrics_collection scripts
447
        // tslint:disable-next-line: no-floating-promises
448
449
        const script = getGpuMetricsCollectorBashScriptContent(remoteGpuScriptCollectorDir);
        SSHClientUtility.remoteExeCommand(`bash -c '${script}'`, conn);
450

451
        const disposable: Rx.IDisposable = this.timer.subscribe(
Deshui Yu's avatar
Deshui Yu committed
452
453
            async (tick: number) => {
                const cmdresult: RemoteCommandResult = await SSHClientUtility.remoteExeCommand(
454
                    `tail -n 1 ${unixPathJoin(remoteGpuScriptCollectorDir, 'gpu_metrics')}`, conn);
455
                if (cmdresult !== undefined && cmdresult.stdout !== undefined) {
Deshui Yu's avatar
Deshui Yu committed
456
                    rmMeta.gpuSummary = <GPUSummary>JSON.parse(cmdresult.stdout);
457
458
459
460
                    if (rmMeta.gpuSummary.gpuCount === 0) {
                        this.log.warning(`No GPU found on remote machine ${rmMeta.ip}`);
                        this.timer.unsubscribe(disposable);
                    }
Deshui Yu's avatar
Deshui Yu committed
461
462
463
464
465
466
467
468
                }
            }
        );
    }

    private async prepareTrialJob(trialJobId: string): Promise<boolean> {
        const deferred : Deferred<boolean> = new Deferred<boolean>();

469
        if (this.trialConfig === undefined) {
Deshui Yu's avatar
Deshui Yu committed
470
471
            throw new Error('trial config is not initialized');
        }
472
473
474
        if (this.gpuScheduler === undefined) {
            throw new Error('gpuScheduler is not initialized');
        }
Deshui Yu's avatar
Deshui Yu committed
475
476
477
478
        const trialJobDetail: RemoteMachineTrialJobDetail | undefined = this.trialJobsMap.get(trialJobId);
        if (trialJobDetail === undefined) {
            throw new NNIError(NNIErrorNames.INVALID_JOB_DETAIL, `Invalid job detail information for trial job ${trialJobId}`);
        }
479
480
481
        // If job is not WATIING, Don't prepare and resolve true immediately
        if (trialJobDetail.status !== 'WAITING') {
            deferred.resolve(true);
482

483
484
            return deferred.promise;
        }
Deshui Yu's avatar
Deshui Yu committed
485
        // get an ssh client from scheduler
486
        const rmScheduleResult: RemoteMachineScheduleResult = this.gpuScheduler.scheduleMachine(this.trialConfig.gpuNum, trialJobDetail);
Deshui Yu's avatar
Deshui Yu committed
487
488
489
490
491
        if (rmScheduleResult.resultType === ScheduleResultType.REQUIRE_EXCEED_TOTAL) {
            const errorMessage : string = `Required GPU number ${this.trialConfig.gpuNum} is too large, no machine can meet`;
            this.log.error(errorMessage);
            deferred.reject();
            throw new NNIError(NNIErrorNames.RESOURCE_NOT_AVAILABLE, errorMessage);
492
        } else if (rmScheduleResult.resultType === ScheduleResultType.SUCCEED
Deshui Yu's avatar
Deshui Yu committed
493
494
            && rmScheduleResult.scheduleInfo !== undefined) {
            const rmScheduleInfo : RemoteMachineScheduleInfo = rmScheduleResult.scheduleInfo;
495
            const trialWorkingFolder: string = unixPathJoin(this.remoteExpRootDir, 'trials', trialJobId);
SparkSnail's avatar
SparkSnail committed
496
497
498
499

            trialJobDetail.rmMeta = rmScheduleInfo.rmMeta;

            await this.allocateSSHClientForTrial(trialJobDetail);
Deshui Yu's avatar
Deshui Yu committed
500
            await this.launchTrialOnScheduledMachine(
501
                trialJobId, trialWorkingFolder, trialJobDetail.form, rmScheduleInfo);
Deshui Yu's avatar
Deshui Yu committed
502
503
504

            trialJobDetail.status = 'RUNNING';
            trialJobDetail.url = `file://${rmScheduleInfo.rmMeta.ip}:${trialWorkingFolder}`;
505
            trialJobDetail.startTime = Date.now();
Deshui Yu's avatar
Deshui Yu committed
506

507
            this.trialJobsMap.set(trialJobId, trialJobDetail);
Deshui Yu's avatar
Deshui Yu committed
508
            deferred.resolve(true);
509
        } else if (rmScheduleResult.resultType === ScheduleResultType.TMP_NO_AVAILABLE_GPU) {
Deshui Yu's avatar
Deshui Yu committed
510
511
512
            this.log.info(`Right now no available GPU can be allocated for trial ${trialJobId}, will try to schedule later`);
            deferred.resolve(false);
        } else {
513
            deferred.reject(`Invalid schedule resutl type: ${rmScheduleResult.resultType}`);
Deshui Yu's avatar
Deshui Yu committed
514
515
516
517
518
519
520
        }

        return deferred.promise;
    }

    private async launchTrialOnScheduledMachine(trialJobId: string, trialWorkingFolder: string, form: TrialJobApplicationForm,
                                                rmScheduleInfo: RemoteMachineScheduleInfo): Promise<void> {
521
        if (this.trialConfig === undefined) {
Deshui Yu's avatar
Deshui Yu committed
522
523
524
            throw new Error('trial config is not initialized');
        }
        const cuda_visible_device: string = rmScheduleInfo.cuda_visible_device;
SparkSnail's avatar
SparkSnail committed
525
        const sshClient: Client | undefined = this.trialSSHClientMap.get(trialJobId);
526
527
528
529
530
531
        if (sshClient === undefined) {
            assert(false, 'sshClient is undefined.');

            // for lint
            return;
        }
532
533
534
535
536
        const trialJobDetail: RemoteMachineTrialJobDetail | undefined = this.trialJobsMap.get(trialJobId);
        if (trialJobDetail === undefined) {
            throw new Error(`Can not get trial job detail for job: ${trialJobId}`);
        }

Deshui Yu's avatar
Deshui Yu committed
537
538
539
        const trialLocalTempFolder: string = path.join(this.expRootDir, 'trials-local', trialJobId);

        await SSHClientUtility.remoteExeCommand(`mkdir -p ${trialWorkingFolder}`, sshClient);
540
        await SSHClientUtility.remoteExeCommand(`mkdir -p ${unixPathJoin(trialWorkingFolder, '.nni')}`, sshClient);
Deshui Yu's avatar
Deshui Yu committed
541
542
543

        // RemoteMachineRunShellFormat is the run shell format string,
        // See definition in remoteMachineData.ts
SparkSnail's avatar
SparkSnail committed
544
545
546
547

        let command: string;
        // Set CUDA_VISIBLE_DEVICES environment variable based on cuda_visible_device
        // If no valid cuda_visible_device is defined, set CUDA_VISIBLE_DEVICES to empty string to hide GPU device
SparkSnail's avatar
SparkSnail committed
548
549
550
        // If gpuNum is undefined, will not set CUDA_VISIBLE_DEVICES in script
        if (this.trialConfig.gpuNum === undefined) {
            command = this.trialConfig.command;
SparkSnail's avatar
SparkSnail committed
551
        } else {
SparkSnail's avatar
SparkSnail committed
552
553
554
555
556
            if (typeof cuda_visible_device === 'string' && cuda_visible_device.length > 0) {
                command = `CUDA_VISIBLE_DEVICES=${cuda_visible_device} ${this.trialConfig.command}`;
            } else {
                command = `CUDA_VISIBLE_DEVICES=" " ${this.trialConfig.command}`;
            }
SparkSnail's avatar
SparkSnail committed
557
        }
558
559
560
        // tslint:disable-next-line: strict-boolean-expressions
        const nniManagerIp: string = this.nniManagerIpConfig ? this.nniManagerIpConfig.nniManagerIp : getIPV4Address();
        if (this.remoteRestServerPort === undefined) {
SparkSnail's avatar
SparkSnail committed
561
562
563
            const restServer: RemoteMachineJobRestServer = component.get(RemoteMachineJobRestServer);
            this.remoteRestServerPort = restServer.clusterRestServerPort;
        }
564
        const version: string = this.versionCheck ? await getVersion() : '';
SparkSnail's avatar
SparkSnail committed
565
566
567
        const runScriptTrialContent: string = String.Format(
            REMOTEMACHINE_TRIAL_COMMAND_FORMAT,
            trialWorkingFolder,
Deshui Yu's avatar
Deshui Yu committed
568
569
            trialWorkingFolder,
            trialJobId,
SparkSnail's avatar
SparkSnail committed
570
            getExperimentId(),
571
            trialJobDetail.form.sequenceId.toString(),
572
            this.isMultiPhase,
573
            unixPathJoin(trialWorkingFolder, '.nni', 'jobpid'),
SparkSnail's avatar
SparkSnail committed
574
575
576
            command,
            nniManagerIp,
            this.remoteRestServerPort,
577
            version,
SparkSnail's avatar
SparkSnail committed
578
            this.logCollection,
579
            unixPathJoin(trialWorkingFolder, '.nni', 'code')
580
        );
Deshui Yu's avatar
Deshui Yu committed
581
582

        //create tmp trial working folder locally.
583
        await execMkdir(path.join(trialLocalTempFolder, '.nni'));
Deshui Yu's avatar
Deshui Yu committed
584

SparkSnail's avatar
SparkSnail committed
585
        //create tmp trial working folder locally.
SparkSnail's avatar
SparkSnail committed
586
        await execCopydir(this.trialConfig.codeDir, trialLocalTempFolder);
SparkSnail's avatar
SparkSnail committed
587
588
589
        const installScriptContent : string = CONTAINER_INSTALL_NNI_SHELL_FORMAT;
        // Write NNI installation file to local tmp files
        await fs.promises.writeFile(path.join(trialLocalTempFolder, 'install_nni.sh'), installScriptContent, { encoding: 'utf8' });
590
        // Write file content ( run.sh and parameter.cfg ) to local tmp files
SparkSnail's avatar
SparkSnail committed
591
        await fs.promises.writeFile(path.join(trialLocalTempFolder, 'run.sh'), runScriptTrialContent, { encoding: 'utf8' });
chicm-ms's avatar
chicm-ms committed
592
        await this.writeParameterFile(trialJobId, form.hyperParameters, rmScheduleInfo.rmMeta);
Deshui Yu's avatar
Deshui Yu committed
593
        // Copy files in codeDir to remote working directory
SparkSnail's avatar
SparkSnail committed
594
        await SSHClientUtility.copyDirectoryToRemote(trialLocalTempFolder, trialWorkingFolder, sshClient, this.remoteOS);
Deshui Yu's avatar
Deshui Yu committed
595
        // Execute command in remote machine
596
        // tslint:disable-next-line: no-floating-promises
597
        SSHClientUtility.remoteExeCommand(`bash ${unixPathJoin(trialWorkingFolder, 'run.sh')}`, sshClient);
Deshui Yu's avatar
Deshui Yu committed
598
599
600
601
602
603
604
605
606
607
608
609
610
611
    }

    private getRmMetaByHost(host: string): RemoteMachineMeta {
        for (const [rmMeta, client] of this.machineSSHClientMap.entries()) {
            if (rmMeta.ip === host) {
                return rmMeta;
            }
        }
        throw new Error(`Host not found: ${host}`);
    }

    private async updateTrialJobStatus(trialJob: RemoteMachineTrialJobDetail, sshClient: Client): Promise<TrialJobDetail> {
        const deferred: Deferred<TrialJobDetail> = new Deferred<TrialJobDetail>();
        const jobpidPath: string = this.getJobPidPath(trialJob.id);
612
        const trialReturnCodeFilePath: string = unixPathJoin(this.remoteExpRootDir, 'trials', trialJob.id, '.nni', 'code');
Deshui Yu's avatar
Deshui Yu committed
613
614
615
616
617
618
        try {
            const killResult: number = (await SSHClientUtility.remoteExeCommand(`kill -0 \`cat ${jobpidPath}\``, sshClient)).exitCode;
            // if the process of jobpid is not alive any more
            if (killResult !== 0) {
                const trailReturnCode: string = await SSHClientUtility.getRemoteFileContent(trialReturnCodeFilePath, sshClient);
                this.log.debug(`trailjob ${trialJob.id} return code: ${trailReturnCode}`);
619
620
621
                const match: RegExpMatchArray | null = trailReturnCode.trim()
                  .match(/^(\d+)\s+(\d+)$/);
                if (match !== null) {
Deshui Yu's avatar
Deshui Yu committed
622
623
624
625
626
                    const { 1: code, 2: timestamp } = match;
                    // Update trial job's status based on result code
                    if (parseInt(code, 10) === 0) {
                        trialJob.status = 'SUCCEEDED';
                    } else {
627
628
629
630
631
632
                        // isEarlyStopped is never set, mean it's not cancelled by NNI, so if the process's exit code >0, mark it as FAILED
                        if (trialJob.isEarlyStopped === undefined) {
                            trialJob.status = 'FAILED';
                        } else {
                            trialJob.status = getJobCancelStatus(trialJob.isEarlyStopped);
                        }
Deshui Yu's avatar
Deshui Yu committed
633
                    }
634
                    trialJob.endTime = parseInt(timestamp, 10);
SparkSnail's avatar
SparkSnail committed
635
                    this.releaseTrialSSHClient(trialJob);
Deshui Yu's avatar
Deshui Yu committed
636
                }
chicm-ms's avatar
chicm-ms committed
637
                this.log.debug(`trailJob status update: ${trialJob.id}, ${trialJob.status}`);
Deshui Yu's avatar
Deshui Yu committed
638
639
640
641
642
643
644
645
646
647
648
            }
            deferred.resolve(trialJob);
        } catch (error) {
            this.log.error(`Update job status exception, error is ${error.message}`);
            if (error instanceof NNIError && error.name === NNIErrorNames.NOT_FOUND) {
                deferred.resolve(trialJob);
            } else {
                trialJob.status = 'UNKNOWN';
                deferred.resolve(trialJob);
            }
        }
649

Deshui Yu's avatar
Deshui Yu committed
650
651
652
        return deferred.promise;
    }

SparkSnail's avatar
SparkSnail committed
653
    private getRemoteScriptsPath(userName: string): string {
654
        return unixPathJoin(getRemoteTmpDir(this.remoteOS), userName, 'nni', 'scripts');
Deshui Yu's avatar
Deshui Yu committed
655
656
657
    }

    private getHostJobRemoteDir(jobId: string): string {
658
        return unixPathJoin(this.remoteExpRootDir, 'hostjobs', jobId);
Deshui Yu's avatar
Deshui Yu committed
659
660
    }

661
    private getRemoteExperimentRootDir(): string {
662
        return unixPathJoin(getRemoteTmpDir(this.remoteOS), 'nni', 'experiments', getExperimentId());
Deshui Yu's avatar
Deshui Yu committed
663
664
    }

SparkSnail's avatar
SparkSnail committed
665
666
667
668
    public get MetricsEmitter() : EventEmitter {
        return this.metricsEmitter;
    }

Deshui Yu's avatar
Deshui Yu committed
669
670
671
672
673
674
675
    private getJobPidPath(jobId: string): string {
        const trialJobDetail: RemoteMachineTrialJobDetail | undefined = this.trialJobsMap.get(jobId);
        if (trialJobDetail === undefined) {
            throw new NNIError(NNIErrorNames.INVALID_JOB_DETAIL, `Invalid job detail information for trial job ${jobId}`);
        }

        let jobpidPath: string;
676
        jobpidPath = unixPathJoin(trialJobDetail.workingDirectory, '.nni', 'jobpid');
Deshui Yu's avatar
Deshui Yu committed
677
678
679

        return jobpidPath;
    }
chicm-ms's avatar
chicm-ms committed
680
681

    private async writeParameterFile(trialJobId: string, hyperParameters: HyperParameters, rmMeta: RemoteMachineMeta): Promise<void> {
SparkSnail's avatar
SparkSnail committed
682
        const sshClient: Client | undefined = this.trialSSHClientMap.get(trialJobId);
chicm-ms's avatar
chicm-ms committed
683
684
685
686
        if (sshClient === undefined) {
            throw new Error('sshClient is undefined.');
        }

687
        const trialWorkingFolder: string = unixPathJoin(this.remoteExpRootDir, 'trials', trialJobId);
chicm-ms's avatar
chicm-ms committed
688
689
        const trialLocalTempFolder: string = path.join(this.expRootDir, 'trials-local', trialJobId);

690
        const fileName: string = generateParamFileName(hyperParameters);
chicm-ms's avatar
chicm-ms committed
691
692
693
        const localFilepath: string = path.join(trialLocalTempFolder, fileName);
        await fs.promises.writeFile(localFilepath, hyperParameters.value, { encoding: 'utf8' });

694
        await SSHClientUtility.copyFileToRemote(localFilepath, unixPathJoin(trialWorkingFolder, fileName), sshClient);
chicm-ms's avatar
chicm-ms committed
695
    }
Deshui Yu's avatar
Deshui Yu committed
696
697
698
}

export { RemoteMachineTrainingService };