remoteMachineTrainingService.ts 34.8 KB
Newer Older
Deshui Yu's avatar
Deshui Yu committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
/**
 * Copyright (c) Microsoft Corporation
 * All rights reserved.
 *
 * MIT License
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
 * documentation files (the "Software"), to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and
 * to permit persons to whom the Software is furnished to do so, subject to the following conditions:
 * The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING
 * BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,
 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 */

'use strict';

22
import * as assert from 'assert';
Deshui Yu's avatar
Deshui Yu committed
23
24
25
26
27
import * as cpp from 'child-process-promise';
import { EventEmitter } from 'events';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
28
import { Client, ConnectConfig } from 'ssh2';
Deshui Yu's avatar
Deshui Yu committed
29
30
31
import { Deferred } from 'ts-deferred';
import { String } from 'typescript-string-operations';
import * as component from '../../common/component';
SparkSnail's avatar
SparkSnail committed
32
import { NNIError, NNIErrorNames } from '../../common/errors';
33
import { getExperimentId, getInitTrialSequenceId } from '../../common/experimentStartupInfo';
Deshui Yu's avatar
Deshui Yu committed
34
35
36
import { getLogger, Logger } from '../../common/log';
import { ObservableTimer } from '../../common/observableTimer';
import {
SparkSnail's avatar
SparkSnail committed
37
    HostJobApplicationForm, HyperParameters, JobApplicationForm, TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric, NNIManagerIpConfig
Deshui Yu's avatar
Deshui Yu committed
38
} from '../../common/trainingService';
SparkSnail's avatar
SparkSnail committed
39
import { delay, generateParamFileName, getExperimentRootDir, uniqueString, getJobCancelStatus, getRemoteTmpDir,getIPV4Address  } from '../../common/utils';
Deshui Yu's avatar
Deshui Yu committed
40
import { GPUSummary } from '../common/gpuData';
41
42
import { TrialConfig } from '../common/trialConfig';
import { TrialConfigMetadataKey } from '../common/trialConfigMetadataKey';
Deshui Yu's avatar
Deshui Yu committed
43
44
import { GPUScheduler } from './gpuScheduler';
import {
45
    HOST_JOB_SHELL_FORMAT, RemoteCommandResult, RemoteMachineMeta,
SparkSnail's avatar
SparkSnail committed
46
    RemoteMachineScheduleInfo, RemoteMachineScheduleResult, SSHClient, SSHClientManager,
SparkSnail's avatar
SparkSnail committed
47
48
    RemoteMachineTrialJobDetail, ScheduleResultType, REMOTEMACHINE_TRIAL_COMMAND_FORMAT, 
    GPU_COLLECTOR_FORMAT
Deshui Yu's avatar
Deshui Yu committed
49
50
} from './remoteMachineData';
import { SSHClientUtility } from './sshClientUtility';
51
import { validateCodeDir } from '../common/util';
SparkSnail's avatar
SparkSnail committed
52
53
54
import { RemoteMachineJobRestServer } from './remoteMachineJobRestServer';
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData';
import { mkDirP } from '../../common/utils';
Deshui Yu's avatar
Deshui Yu committed
55
56
57
58

/**
 * Training Service implementation for Remote Machine (Linux)
 */
SparkSnail's avatar
SparkSnail committed
59
@component.Singleton
Deshui Yu's avatar
Deshui Yu committed
60
class RemoteMachineTrainingService implements TrainingService {
SparkSnail's avatar
SparkSnail committed
61
62
    private machineSSHClientMap: Map<RemoteMachineMeta, SSHClientManager>; //machine ssh client map
    private trialSSHClientMap: Map<string, Client>; //trial ssh client map
Deshui Yu's avatar
Deshui Yu committed
63
    private trialJobsMap: Map<string, RemoteMachineTrialJobDetail>;
SparkSnail's avatar
SparkSnail committed
64
    private readonly MAX_TRIAL_NUMBER_PER_SSHCONNECTION: number = 5 // every ssh client has a max trial concurrency number
Deshui Yu's avatar
Deshui Yu committed
65
66
    private expRootDir: string;
    private remoteExpRootDir: string;
67
    private trialConfig: TrialConfig | undefined;
Deshui Yu's avatar
Deshui Yu committed
68
69
70
71
72
73
    private gpuScheduler: GPUScheduler;
    private jobQueue: string[];
    private timer: ObservableTimer;
    private stopping: boolean = false;
    private metricsEmitter: EventEmitter;
    private log: Logger;
74
    private isMultiPhase: boolean = false;
75
    private trialSequenceId: number;
SparkSnail's avatar
SparkSnail committed
76
    private remoteRestServerPort?: number;
77
    private readonly remoteOS: string;
SparkSnail's avatar
SparkSnail committed
78
    private nniManagerIpConfig?: NNIManagerIpConfig;
Deshui Yu's avatar
Deshui Yu committed
79
80

    constructor(@component.Inject timer: ObservableTimer) {
81
        this.remoteOS = 'linux';
Deshui Yu's avatar
Deshui Yu committed
82
83
        this.metricsEmitter = new EventEmitter();
        this.trialJobsMap = new Map<string, RemoteMachineTrialJobDetail>();
SparkSnail's avatar
SparkSnail committed
84
85
        this.trialSSHClientMap = new Map<string, Client>();
        this.machineSSHClientMap = new Map<RemoteMachineMeta, SSHClientManager>();
Deshui Yu's avatar
Deshui Yu committed
86
87
88
        this.gpuScheduler = new GPUScheduler(this.machineSSHClientMap);
        this.jobQueue = [];
        this.expRootDir = getExperimentRootDir();
89
        this.remoteExpRootDir = this.getRemoteExperimentRootDir();
Deshui Yu's avatar
Deshui Yu committed
90
91
        this.timer = timer;
        this.log = getLogger();
92
        this.trialSequenceId = -1;
chicm-ms's avatar
chicm-ms committed
93
        this.log.info('Construct remote machine training service.');
Deshui Yu's avatar
Deshui Yu committed
94
95
96
97
98
99
    }

    /**
     * Loop to launch trial jobs and collect trial metrics
     */
    public async run(): Promise<void> {
SparkSnail's avatar
SparkSnail committed
100
101
        const restServer: RemoteMachineJobRestServer = component.get(RemoteMachineJobRestServer);
        await restServer.start();
chicm-ms's avatar
chicm-ms committed
102
        this.log.info('Run remote machine training service.');
Deshui Yu's avatar
Deshui Yu committed
103
104
        while (!this.stopping) {
            while (this.jobQueue.length > 0) {
SparkSnail's avatar
SparkSnail committed
105
                this.updateGpuReservation();
Deshui Yu's avatar
Deshui Yu committed
106
107
108
109
110
111
                const trialJobId: string = this.jobQueue[0];
                const prepareResult : boolean = await this.prepareTrialJob(trialJobId);
                if (prepareResult) {
                    // Remove trial job with trialJobId from job queue
                    this.jobQueue.shift();
                } else {
112
                    // Break the while loop since no GPU resource is available right now,
Deshui Yu's avatar
Deshui Yu committed
113
114
115
                    // Wait to schedule job in next time iteration
                    break;
                }
116
            }
Deshui Yu's avatar
Deshui Yu committed
117
118
            await delay(3000);
        }
chicm-ms's avatar
chicm-ms committed
119
        this.log.info('Remote machine training service exit.');
Deshui Yu's avatar
Deshui Yu committed
120
    }
SparkSnail's avatar
SparkSnail committed
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
    
    /**
     * give trial a ssh connection
     * @param trial 
     */
    public async allocateSSHClientForTrial(trial: RemoteMachineTrialJobDetail): Promise<void> {
        const deferred: Deferred<void> = new Deferred<void>();
        if(!trial.rmMeta) {
            throw new Error(`rmMeta not set in trial ${trial.id}`);
        }
        let sshClientManager: SSHClientManager | undefined = this.machineSSHClientMap.get(trial.rmMeta);
        if(!sshClientManager) {
            throw new Error(`remoteSSHClient not initialized`);
        }
        let sshClient: Client = await sshClientManager.getAvailableSSHClient();
        this.trialSSHClientMap.set(trial.id, sshClient);
        deferred.resolve();
        return deferred.promise;
    }
    
    /**
     * If a trial is finished, release the connection resource
     * @param trial 
     */
    public releaseTrialSSHClient(trial: RemoteMachineTrialJobDetail): void {
        if(!trial.rmMeta) {
            throw new Error(`rmMeta not set in trial ${trial.id}`);
        }
        let sshClientManager: SSHClientManager | undefined = this.machineSSHClientMap.get(trial.rmMeta);
        if(!sshClientManager) {
            throw new Error(`sshClientManager not initialized`);
        }
        sshClientManager.releaseConnection(this.trialSSHClientMap.get(trial.id));
    }
Deshui Yu's avatar
Deshui Yu committed
155
156
157
158

    /**
     * List submitted trial jobs
     */
159
    public async listTrialJobs(): Promise<TrialJobDetail[]> {
Deshui Yu's avatar
Deshui Yu committed
160
161
162
        const jobs: TrialJobDetail[] = [];
        const deferred: Deferred<TrialJobDetail[]> = new Deferred<TrialJobDetail[]>();

163
        for (const [key, value] of this.trialJobsMap) { 
Deshui Yu's avatar
Deshui Yu committed
164
165
166
            if (value.form.jobType === 'TRIAL') {
                jobs.push(await this.getTrialJob(key));
            }
167
        };
Deshui Yu's avatar
Deshui Yu committed
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
        deferred.resolve(jobs);

        return deferred.promise;
    }

    /**
     * Get trial job detail information
     * @param trialJobId ID of trial job
     */
    public async getTrialJob(trialJobId: string): Promise<TrialJobDetail> {
        const trialJob: RemoteMachineTrialJobDetail | undefined = this.trialJobsMap.get(trialJobId);
        if (!trialJob) {
            throw new NNIError(NNIErrorNames.NOT_FOUND, `trial job id ${trialJobId} not found`);
        }
        //TO DO: add another job status, and design new job status change logic
        if (trialJob.status === 'RUNNING' || trialJob.status === 'UNKNOWN') {
            // Get ssh client where the job is running
            if (trialJob.rmMeta === undefined) {
                throw new Error(`rmMeta not set for submitted job ${trialJobId}`);
            }
SparkSnail's avatar
SparkSnail committed
188
            const sshClient: Client | undefined  = this.trialSSHClientMap.get(trialJob.id);
Deshui Yu's avatar
Deshui Yu committed
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
            if (!sshClient) {
                throw new Error(`Invalid job id: ${trialJobId}, cannot find ssh client`);
            }

            return this.updateTrialJobStatus(trialJob, sshClient);
        } else {
            return trialJob;
        }
    }

    /**
     * Add job metrics listener
     * @param listener callback listener
     */
    public addTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void {
        this.metricsEmitter.on('metric', listener);
    }

    /**
     * Remove job metrics listener
     * @param listener callback listener
     */
    public removeTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void {
        this.metricsEmitter.off('metric', listener);
    }

    /**
     * Submit trial job
     * @param form trial job description form
     */
SparkSnail's avatar
SparkSnail committed
219
    public async submitTrialJob(form: JobApplicationForm): Promise<TrialJobDetail> {
Deshui Yu's avatar
Deshui Yu committed
220
221
222
223
224
225
226
227
228
229
230
231
232
233
        if (!this.trialConfig) {
            throw new Error('trial config is not initialized');
        }

        if (form.jobType === 'HOST') {
            return this.runHostJob(<HostJobApplicationForm>form);
        } else if (form.jobType === 'TRIAL') {
            // Generate trial job id(random)
            const trialJobId: string = uniqueString(5);
            const trialWorkingFolder: string = path.join(this.remoteExpRootDir, 'trials', trialJobId);

            const trialJobDetail: RemoteMachineTrialJobDetail = new RemoteMachineTrialJobDetail(
                trialJobId,
                'WAITING',
234
                Date.now(),
Deshui Yu's avatar
Deshui Yu committed
235
                trialWorkingFolder,
236
237
238
                form,
                this.generateSequenceId()
            );
Deshui Yu's avatar
Deshui Yu committed
239
240
            this.jobQueue.push(trialJobId);
            this.trialJobsMap.set(trialJobId, trialJobDetail);
241

Deshui Yu's avatar
Deshui Yu committed
242
243
244
245
246
247
            return Promise.resolve(trialJobDetail);
        } else {
            return Promise.reject(new Error(`Job form not supported: ${JSON.stringify(form)}, jobType should be HOST or TRIAL.`));
        }
    }

248
249
250
251
252
    /**
     * Update trial job for multi-phase
     * @param trialJobId trial job id
     * @param form job application form
     */
chicm-ms's avatar
chicm-ms committed
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
    public async updateTrialJob(trialJobId: string, form: JobApplicationForm): Promise<TrialJobDetail> {
        const trialJobDetail: undefined | TrialJobDetail = this.trialJobsMap.get(trialJobId);
        if (trialJobDetail === undefined) {
            throw new Error(`updateTrialJob failed: ${trialJobId} not found`);
        }
        if (form.jobType === 'TRIAL') {
            const rmMeta: RemoteMachineMeta | undefined = (<RemoteMachineTrialJobDetail>trialJobDetail).rmMeta;
            if (rmMeta !== undefined) {
                await this.writeParameterFile(trialJobId, (<TrialJobApplicationForm>form).hyperParameters, rmMeta);
            } else {
                throw new Error(`updateTrialJob failed: ${trialJobId} rmMeta not found`);
            }
        } else {
            throw new Error(`updateTrialJob failed: jobType ${form.jobType} not supported.`);
        }

        return trialJobDetail;
270
    }
SparkSnail's avatar
SparkSnail committed
271
272
273
274
275
276
277
278
279
280
281
    
    /**
     * remove gpu reversion when job is not running
     */
    private updateGpuReservation() {
        for (const [key, value] of this.trialJobsMap) { 
            if(!['WAITING', 'RUNNING'].includes(value.status)) {
                this.gpuScheduler.removeGpuReservation(value.id, value.rmMeta);
            }
        };
    }
282
283
284
285
286

    /**
     * Is multiphase job supported in current training service
     */
    public get isMultiPhaseJobSupported(): boolean {
287
        return true;
288
289
    }

Deshui Yu's avatar
Deshui Yu committed
290
291
292
293
    /**
     * Cancel trial job
     * @param trialJobId ID of trial job
     */
QuanluZhang's avatar
QuanluZhang committed
294
    public async cancelTrialJob(trialJobId: string, isEarlyStopped: boolean = false): Promise<void> {
Deshui Yu's avatar
Deshui Yu committed
295
296
297
298
299
300
301
302
303
        const deferred: Deferred<void> = new Deferred<void>();
        const trialJob: RemoteMachineTrialJobDetail | undefined = this.trialJobsMap.get(trialJobId);
        if (!trialJob) {
            deferred.reject();
            throw new Error(`trial job id ${trialJobId} not found`);
        }

        // Remove the job with trialJobId from job queue
        const index : number = this.jobQueue.indexOf(trialJobId);
304
        if (index >= 0) {
Deshui Yu's avatar
Deshui Yu committed
305
306
307
308
309
310
            this.jobQueue.splice(index, 1);
        }

        // Get ssh client where the job is running
        if (trialJob.rmMeta !== undefined) {
            // If the trial job is already scheduled, check its status and kill the trial process in remote machine
SparkSnail's avatar
SparkSnail committed
311
            const sshClient: Client | undefined = this.trialSSHClientMap.get(trialJob.id);
Deshui Yu's avatar
Deshui Yu committed
312
313
314
315
316
317
318
            if (!sshClient) {
                deferred.reject();
                throw new Error(`Invalid job id ${trialJobId}, cannot find ssh client`);
            }

            const jobpidPath: string = this.getJobPidPath(trialJob.id);
            try {
319
320
                // Mark the toEarlyStop tag here
                trialJob.isEarlyStopped = isEarlyStopped;
Deshui Yu's avatar
Deshui Yu committed
321
                await SSHClientUtility.remoteExeCommand(`pkill -P \`cat ${jobpidPath}\``, sshClient);
SparkSnail's avatar
SparkSnail committed
322
                this.releaseTrialSSHClient(trialJob);
Deshui Yu's avatar
Deshui Yu committed
323
324
325
326
327
328
            } catch (error) {
                // Not handle the error since pkill failed will not impact trial job's current status
                this.log.error(`remoteTrainingService.cancelTrialJob: ${error.message}`);
            }
        } else {
            // Job is not scheduled yet, set status to 'USER_CANCELLED' directly
QuanluZhang's avatar
QuanluZhang committed
329
330
            assert(isEarlyStopped === false, 'isEarlyStopped is not supposed to be true here.');
            trialJob.status = getJobCancelStatus(isEarlyStopped);
Deshui Yu's avatar
Deshui Yu committed
331
332
333
334
335
336
337
338
339
340
341
342
        }
    }

    /**
     * Set culster metadata
     * @param key metadata key
     * //1. MACHINE_LIST -- create ssh client connect of machine list
     * //2. TRIAL_CONFIG -- trial configuration
     * @param value metadata value
     */
    public async setClusterMetadata(key: string, value: string): Promise<void> {
        switch (key) {
SparkSnail's avatar
SparkSnail committed
343
344
345
            case TrialConfigMetadataKey.NNI_MANAGER_IP:
                this.nniManagerIpConfig = <NNIManagerIpConfig>JSON.parse(value);
                break;
346
            case TrialConfigMetadataKey.MACHINE_LIST:
Deshui Yu's avatar
Deshui Yu committed
347
                await this.setupConnections(value);
SparkSnail's avatar
SparkSnail committed
348
349
                //remove local temp files
                await cpp.exec(`rm -rf ${this.getLocalGpuMetricCollectorDir()}`);
Deshui Yu's avatar
Deshui Yu committed
350
                break;
351
352
            case TrialConfigMetadataKey.TRIAL_CONFIG:
                const remoteMachineTrailConfig: TrialConfig = <TrialConfig>JSON.parse(value);
Deshui Yu's avatar
Deshui Yu committed
353
354
355
356
357
358
359
360
                // Parse trial config failed, throw Error
                if (!remoteMachineTrailConfig) {
                    throw new Error('trial config parsed failed');
                }
                // codeDir is not a valid directory, throw Error
                if (!fs.lstatSync(remoteMachineTrailConfig.codeDir).isDirectory()) {
                    throw new Error(`codeDir ${remoteMachineTrailConfig.codeDir} is not a directory`);
                }
361
362
363
364
365
366
367
368
369

                // Validate to make sure codeDir doesn't have too many files
                try {
                    await validateCodeDir(remoteMachineTrailConfig.codeDir);
                } catch(error) {
                    this.log.error(error);
                    return Promise.reject(new Error(error));                    
                }

Deshui Yu's avatar
Deshui Yu committed
370
371
                this.trialConfig = remoteMachineTrailConfig;
                break;
372
373
374
            case TrialConfigMetadataKey.MULTI_PHASE:
                this.isMultiPhase = (value === 'true' || value === 'True');
                break;
Deshui Yu's avatar
Deshui Yu committed
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
            default:
                //Reject for unknown keys
                throw new Error(`Uknown key: ${key}`);
        }
    }

    /**
     * Get culster metadata
     * @param key metadata key
     */
    public getClusterMetadata(key: string): Promise<string> {
        const deferred: Deferred<string> = new Deferred<string>();

        return deferred.promise;
    }
SparkSnail's avatar
SparkSnail committed
390
391
392
393
394
    
    /**
     * cleanup() has a time out of 10s to clean remote connections 
     */
    public async cleanUp(): Promise<void> {
chicm-ms's avatar
chicm-ms committed
395
        this.log.info('Stopping remote machine training service...');
Deshui Yu's avatar
Deshui Yu committed
396
        this.stopping = true;
SparkSnail's avatar
SparkSnail committed
397
398
399
400
401
402
403
404
        await Promise.race([delay(10000), this.cleanupConnections()]);
    }
    
    /**
     * stop gpu_metric_collector process in remote machine and remove unused scripts
     */
    private async cleanupConnections(): Promise<void> {
        try{
SparkSnail's avatar
SparkSnail committed
405
            for (const [rmMeta, sshClientManager] of this.machineSSHClientMap.entries()) {
SparkSnail's avatar
SparkSnail committed
406
                let jobpidPath: string = path.join(this.getRemoteScriptsPath(rmMeta.username), 'pid');
SparkSnail's avatar
SparkSnail committed
407
408
409
410
411
412
                let client: Client | undefined = sshClientManager.getFirstSSHClient();
                if(client) {
                    await SSHClientUtility.remoteExeCommand(`pkill -P \`cat ${jobpidPath}\``, client);
                    await SSHClientUtility.remoteExeCommand(`rm -rf ${this.getRemoteScriptsPath(rmMeta.username)}`, client);
                }
                sshClientManager.closeAllSSHClient();
SparkSnail's avatar
SparkSnail committed
413
414
415
416
417
            }
        }catch (error) {
            //ignore error, this function is called to cleanup remote connections when experiment is stopping
            this.log.error(`Cleanup connection exception, error is ${error.message}`);
        }
Deshui Yu's avatar
Deshui Yu committed
418
419

        return Promise.resolve();
SparkSnail's avatar
SparkSnail committed
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
    } 
    
    /**
     * Generate gpu metric collector directory to store temp gpu metric collector script files
     */
    private getLocalGpuMetricCollectorDir(): string {
        let userName: string = path.basename(os.homedir()); //get current user name of os
        return `${os.tmpdir()}/${userName}/nni/scripts/`;
    }

    /**
     * Generate gpu metric collector shell script in local machine, 
     * used to run in remote machine, and will be deleted after uploaded from local. 
     */
    private async generateGpuMetricsCollectorScript(userName: string): Promise<void> {
        let gpuMetricCollectorScriptFolder : string = this.getLocalGpuMetricCollectorDir();
        await cpp.exec(`mkdir -p ${path.join(gpuMetricCollectorScriptFolder, userName)}`);
        //generate gpu_metrics_collector.sh
        let gpuMetricsCollectorScriptPath: string = path.join(gpuMetricCollectorScriptFolder, userName, 'gpu_metrics_collector.sh');
        const remoteGPUScriptsDir: string = this.getRemoteScriptsPath(userName); // This directory is used to store gpu_metrics and pid created by script
        const gpuMetricsCollectorScriptContent: string = String.Format(
            GPU_COLLECTOR_FORMAT, 
            remoteGPUScriptsDir, 
            path.join(remoteGPUScriptsDir, 'pid'), 
        );
        await fs.promises.writeFile(gpuMetricsCollectorScriptPath, gpuMetricsCollectorScriptContent, { encoding: 'utf8' });
Deshui Yu's avatar
Deshui Yu committed
446
447
448
    }

    private async setupConnections(machineList: string): Promise<void> {
chicm-ms's avatar
chicm-ms committed
449
        this.log.debug(`Connecting to remote machines: ${machineList}`);
Deshui Yu's avatar
Deshui Yu committed
450
451
452
453
        const deferred: Deferred<void> = new Deferred<void>();
        //TO DO: verify if value's format is wrong, and json parse failed, how to handle error
        const rmMetaList: RemoteMachineMeta[] = <RemoteMachineMeta[]>JSON.parse(machineList);
        let connectedRMNum: number = 0;
SparkSnail's avatar
SparkSnail committed
454

SparkSnail's avatar
SparkSnail committed
455
456
457
458
459
460
461
        rmMetaList.forEach(async (rmMeta: RemoteMachineMeta) => {
            let sshClientManager: SSHClientManager = new SSHClientManager([], this.MAX_TRIAL_NUMBER_PER_SSHCONNECTION, rmMeta);
            let sshClient: Client = await sshClientManager.getAvailableSSHClient();
            this.machineSSHClientMap.set(rmMeta, sshClientManager);
            await this.initRemoteMachineOnConnected(rmMeta, sshClient);
            if (++connectedRMNum === rmMetaList.length) {
                deferred.resolve();
462
            }
Deshui Yu's avatar
Deshui Yu committed
463
464
465
466
467
468
        });
        return deferred.promise;
    }

    private async initRemoteMachineOnConnected(rmMeta: RemoteMachineMeta, conn: Client): Promise<void> {
        // Create root working directory after ssh connection is ready
SparkSnail's avatar
SparkSnail committed
469
470
        await this.generateGpuMetricsCollectorScript(rmMeta.username); //generate gpu script in local machine first, will copy to remote machine later
        const nniRootDir: string = `${os.tmpdir()}/nni`;
Deshui Yu's avatar
Deshui Yu committed
471
        await SSHClientUtility.remoteExeCommand(`mkdir -p ${this.remoteExpRootDir}`, conn);
472

Deshui Yu's avatar
Deshui Yu committed
473
        // Copy NNI scripts to remote expeirment working directory
SparkSnail's avatar
SparkSnail committed
474
475
476
        const localGpuScriptCollectorDir: string = this.getLocalGpuMetricCollectorDir();
        const remoteGpuScriptCollectorDir: string = this.getRemoteScriptsPath(rmMeta.username); //the directory to store temp scripts in remote machine
        await SSHClientUtility.remoteExeCommand(`mkdir -p ${remoteGpuScriptCollectorDir}`, conn);
Deshui Yu's avatar
Deshui Yu committed
477
        await SSHClientUtility.remoteExeCommand(`chmod 777 ${nniRootDir} ${nniRootDir}/* ${nniRootDir}/scripts/*`, conn);
SparkSnail's avatar
SparkSnail committed
478
479
        //copy gpu_metrics_collector.sh to remote
        await SSHClientUtility.copyFileToRemote(path.join(localGpuScriptCollectorDir, rmMeta.username, 'gpu_metrics_collector.sh'), path.join(remoteGpuScriptCollectorDir, 'gpu_metrics_collector.sh'), conn);
480

Deshui Yu's avatar
Deshui Yu committed
481
        //Begin to execute gpu_metrics_collection scripts
SparkSnail's avatar
SparkSnail committed
482
        SSHClientUtility.remoteExeCommand(`bash ${path.join(remoteGpuScriptCollectorDir, 'gpu_metrics_collector.sh')}`, conn);
483

Deshui Yu's avatar
Deshui Yu committed
484
485
486
        this.timer.subscribe(
            async (tick: number) => {
                const cmdresult: RemoteCommandResult = await SSHClientUtility.remoteExeCommand(
SparkSnail's avatar
SparkSnail committed
487
                    `tail -n 1 ${path.join(remoteGpuScriptCollectorDir, 'gpu_metrics')}`, conn);
Deshui Yu's avatar
Deshui Yu committed
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
                if (cmdresult && cmdresult.stdout) {
                    rmMeta.gpuSummary = <GPUSummary>JSON.parse(cmdresult.stdout);
                }
            }
        );
    }

    private async prepareTrialJob(trialJobId: string): Promise<boolean> {
        const deferred : Deferred<boolean> = new Deferred<boolean>();

        if (!this.trialConfig) {
            throw new Error('trial config is not initialized');
        }
        const trialJobDetail: RemoteMachineTrialJobDetail | undefined = this.trialJobsMap.get(trialJobId);
        if (trialJobDetail === undefined) {
            throw new NNIError(NNIErrorNames.INVALID_JOB_DETAIL, `Invalid job detail information for trial job ${trialJobId}`);
        }
505
506
507
508
509
        // If job is not WATIING, Don't prepare and resolve true immediately
        if (trialJobDetail.status !== 'WAITING') {
            deferred.resolve(true);
            return deferred.promise;
        }
Deshui Yu's avatar
Deshui Yu committed
510
511
512
513
514
515
516
        // get an ssh client from scheduler
        const rmScheduleResult: RemoteMachineScheduleResult = this.gpuScheduler.scheduleMachine(this.trialConfig.gpuNum, trialJobId);
        if (rmScheduleResult.resultType === ScheduleResultType.REQUIRE_EXCEED_TOTAL) {
            const errorMessage : string = `Required GPU number ${this.trialConfig.gpuNum} is too large, no machine can meet`;
            this.log.error(errorMessage);
            deferred.reject();
            throw new NNIError(NNIErrorNames.RESOURCE_NOT_AVAILABLE, errorMessage);
517
        } else if (rmScheduleResult.resultType === ScheduleResultType.SUCCEED
Deshui Yu's avatar
Deshui Yu committed
518
519
520
            && rmScheduleResult.scheduleInfo !== undefined) {
            const rmScheduleInfo : RemoteMachineScheduleInfo = rmScheduleResult.scheduleInfo;
            const trialWorkingFolder: string = path.join(this.remoteExpRootDir, 'trials', trialJobId);
SparkSnail's avatar
SparkSnail committed
521
522
523
524

            trialJobDetail.rmMeta = rmScheduleInfo.rmMeta;

            await this.allocateSSHClientForTrial(trialJobDetail);
Deshui Yu's avatar
Deshui Yu committed
525
526
527
528
529
            await this.launchTrialOnScheduledMachine(
                trialJobId, trialWorkingFolder, <TrialJobApplicationForm>trialJobDetail.form, rmScheduleInfo);

            trialJobDetail.status = 'RUNNING';
            trialJobDetail.url = `file://${rmScheduleInfo.rmMeta.ip}:${trialWorkingFolder}`;
530
            trialJobDetail.startTime = Date.now();
Deshui Yu's avatar
Deshui Yu committed
531
532

            deferred.resolve(true);
533
        } else if (rmScheduleResult.resultType === ScheduleResultType.TMP_NO_AVAILABLE_GPU) {
Deshui Yu's avatar
Deshui Yu committed
534
535
536
            this.log.info(`Right now no available GPU can be allocated for trial ${trialJobId}, will try to schedule later`);
            deferred.resolve(false);
        } else {
537
            deferred.reject(`Invalid schedule resutl type: ${rmScheduleResult.resultType}`);
Deshui Yu's avatar
Deshui Yu committed
538
539
540
541
542
543
544
545
546
547
548
        }

        return deferred.promise;
    }

    private async launchTrialOnScheduledMachine(trialJobId: string, trialWorkingFolder: string, form: TrialJobApplicationForm,
                                                rmScheduleInfo: RemoteMachineScheduleInfo): Promise<void> {
        if (!this.trialConfig) {
            throw new Error('trial config is not initialized');
        }
        const cuda_visible_device: string = rmScheduleInfo.cuda_visible_device;
SparkSnail's avatar
SparkSnail committed
549
        const sshClient: Client | undefined = this.trialSSHClientMap.get(trialJobId);
550
551
552
553
554
555
        if (sshClient === undefined) {
            assert(false, 'sshClient is undefined.');

            // for lint
            return;
        }
556
557
558
559
560
        const trialJobDetail: RemoteMachineTrialJobDetail | undefined = this.trialJobsMap.get(trialJobId);
        if (trialJobDetail === undefined) {
            throw new Error(`Can not get trial job detail for job: ${trialJobId}`);
        }

Deshui Yu's avatar
Deshui Yu committed
561
562
563
564
565
566
567
        const trialLocalTempFolder: string = path.join(this.expRootDir, 'trials-local', trialJobId);

        await SSHClientUtility.remoteExeCommand(`mkdir -p ${trialWorkingFolder}`, sshClient);
        await SSHClientUtility.remoteExeCommand(`mkdir -p ${path.join(trialWorkingFolder, '.nni')}`, sshClient);

        // RemoteMachineRunShellFormat is the run shell format string,
        // See definition in remoteMachineData.ts
SparkSnail's avatar
SparkSnail committed
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585

        let command: string;
        // Set CUDA_VISIBLE_DEVICES environment variable based on cuda_visible_device
        // If no valid cuda_visible_device is defined, set CUDA_VISIBLE_DEVICES to empty string to hide GPU device
        if(typeof cuda_visible_device === 'string' && cuda_visible_device.length > 0) {
            command = `CUDA_VISIBLE_DEVICES=${cuda_visible_device} ${this.trialConfig.command}`;
        } else {
            command = `CUDA_VISIBLE_DEVICES=" " ${this.trialConfig.command}`;
        }
        
        const nniManagerIp = this.nniManagerIpConfig?this.nniManagerIpConfig.nniManagerIp:getIPV4Address();
        if(!this.remoteRestServerPort) {
            const restServer: RemoteMachineJobRestServer = component.get(RemoteMachineJobRestServer);
            this.remoteRestServerPort = restServer.clusterRestServerPort;
        }
        const runScriptTrialContent: string = String.Format(
            REMOTEMACHINE_TRIAL_COMMAND_FORMAT,
            trialWorkingFolder,
Deshui Yu's avatar
Deshui Yu committed
586
587
            trialWorkingFolder,
            trialJobId,
SparkSnail's avatar
SparkSnail committed
588
589
            getExperimentId(),
            trialJobDetail.sequenceId.toString(),
590
            this.isMultiPhase,
SparkSnail's avatar
SparkSnail committed
591
592
593
594
595
596
            path.join(trialWorkingFolder, '.nni', 'jobpid'),
            command,
            nniManagerIp,
            this.remoteRestServerPort,
            path.join(trialWorkingFolder, '.nni', 'code')
        )
Deshui Yu's avatar
Deshui Yu committed
597
598

        //create tmp trial working folder locally.
599
        await cpp.exec(`mkdir -p ${path.join(trialLocalTempFolder, '.nni')}`);
Deshui Yu's avatar
Deshui Yu committed
600

SparkSnail's avatar
SparkSnail committed
601
602
603
604
605
        //create tmp trial working folder locally.
        await cpp.exec(`cp -r ${this.trialConfig.codeDir}/* ${trialLocalTempFolder}`);
        const installScriptContent : string = CONTAINER_INSTALL_NNI_SHELL_FORMAT;
        // Write NNI installation file to local tmp files
        await fs.promises.writeFile(path.join(trialLocalTempFolder, 'install_nni.sh'), installScriptContent, { encoding: 'utf8' });
606
        // Write file content ( run.sh and parameter.cfg ) to local tmp files
SparkSnail's avatar
SparkSnail committed
607
        await fs.promises.writeFile(path.join(trialLocalTempFolder, 'run.sh'), runScriptTrialContent, { encoding: 'utf8' });
chicm-ms's avatar
chicm-ms committed
608
        await this.writeParameterFile(trialJobId, form.hyperParameters, rmScheduleInfo.rmMeta);
Deshui Yu's avatar
Deshui Yu committed
609
        // Copy files in codeDir to remote working directory
SparkSnail's avatar
SparkSnail committed
610
        await SSHClientUtility.copyDirectoryToRemote(trialLocalTempFolder, trialWorkingFolder, sshClient, this.remoteOS);
Deshui Yu's avatar
Deshui Yu committed
611
612
613
614
615
616
        // Execute command in remote machine
        SSHClientUtility.remoteExeCommand(`bash ${path.join(trialWorkingFolder, 'run.sh')}`, sshClient);
    }

    private async runHostJob(form: HostJobApplicationForm): Promise<TrialJobDetail> {
        const rmMeta: RemoteMachineMeta = this.getRmMetaByHost(form.host);
SparkSnail's avatar
SparkSnail committed
617
618
        const sshClientManager: SSHClientManager | undefined = this.machineSSHClientMap.get(rmMeta);
        if (sshClientManager === undefined) {
Deshui Yu's avatar
Deshui Yu committed
619
620
            throw new Error('sshClient not found.');
        }
SparkSnail's avatar
SparkSnail committed
621
        let sshClient: Client = sshClientManager.getFirstSSHClient();
Deshui Yu's avatar
Deshui Yu committed
622
623
624
625
626
627
        const jobId: string = uniqueString(5);
        const localDir: string = path.join(this.expRootDir, 'hostjobs-local', jobId);
        const remoteDir: string = this.getHostJobRemoteDir(jobId);
        await cpp.exec(`mkdir -p ${localDir}`);
        await SSHClientUtility.remoteExeCommand(`mkdir -p ${remoteDir}`, sshClient);
        const runScriptContent: string = String.Format(
628
            HOST_JOB_SHELL_FORMAT, remoteDir, path.join(remoteDir, 'jobpid'), form.cmd, path.join(remoteDir, 'code')
Deshui Yu's avatar
Deshui Yu committed
629
630
631
632
633
634
        );
        await fs.promises.writeFile(path.join(localDir, 'run.sh'), runScriptContent, { encoding: 'utf8' });
        await SSHClientUtility.copyFileToRemote(
            path.join(localDir, 'run.sh'), path.join(remoteDir, 'run.sh'), sshClient);
        SSHClientUtility.remoteExeCommand(`bash ${path.join(remoteDir, 'run.sh')}`, sshClient);

635
636
637
        const jobDetail: RemoteMachineTrialJobDetail =  new RemoteMachineTrialJobDetail(
            jobId, 'RUNNING', Date.now(), remoteDir, form, this.generateSequenceId()
        );
Deshui Yu's avatar
Deshui Yu committed
638
        jobDetail.rmMeta = rmMeta;
639
        jobDetail.startTime = Date.now();
Deshui Yu's avatar
Deshui Yu committed
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
        this.trialJobsMap.set(jobId, jobDetail);
        this.log.debug(`runHostJob: return: ${JSON.stringify(jobDetail)} `);

        return jobDetail;
    }

    private getRmMetaByHost(host: string): RemoteMachineMeta {
        for (const [rmMeta, client] of this.machineSSHClientMap.entries()) {
            if (rmMeta.ip === host) {
                return rmMeta;
            }
        }
        throw new Error(`Host not found: ${host}`);
    }

    private async updateTrialJobStatus(trialJob: RemoteMachineTrialJobDetail, sshClient: Client): Promise<TrialJobDetail> {
        const deferred: Deferred<TrialJobDetail> = new Deferred<TrialJobDetail>();
        const jobpidPath: string = this.getJobPidPath(trialJob.id);
        const trialReturnCodeFilePath: string = path.join(this.remoteExpRootDir, 'trials', trialJob.id, '.nni', 'code');
        try {
            const killResult: number = (await SSHClientUtility.remoteExeCommand(`kill -0 \`cat ${jobpidPath}\``, sshClient)).exitCode;
            // if the process of jobpid is not alive any more
            if (killResult !== 0) {
                const trailReturnCode: string = await SSHClientUtility.getRemoteFileContent(trialReturnCodeFilePath, sshClient);
                this.log.debug(`trailjob ${trialJob.id} return code: ${trailReturnCode}`);
                const match: RegExpMatchArray | null = trailReturnCode.trim().match(/^(\d+)\s+(\d+)$/);
                if (match) {
                    const { 1: code, 2: timestamp } = match;
                    // Update trial job's status based on result code
                    if (parseInt(code, 10) === 0) {
                        trialJob.status = 'SUCCEEDED';
                    } else {
672
673
674
675
676
677
                        // isEarlyStopped is never set, mean it's not cancelled by NNI, so if the process's exit code >0, mark it as FAILED
                        if (trialJob.isEarlyStopped === undefined) {
                            trialJob.status = 'FAILED';
                        } else {
                            trialJob.status = getJobCancelStatus(trialJob.isEarlyStopped);
                        }
Deshui Yu's avatar
Deshui Yu committed
678
                    }
679
                    trialJob.endTime = parseInt(timestamp, 10);
SparkSnail's avatar
SparkSnail committed
680
                    this.releaseTrialSSHClient(trialJob);
Deshui Yu's avatar
Deshui Yu committed
681
                }
chicm-ms's avatar
chicm-ms committed
682
                this.log.debug(`trailJob status update: ${trialJob.id}, ${trialJob.status}`);
Deshui Yu's avatar
Deshui Yu committed
683
684
685
686
687
688
689
690
691
692
693
694
695
696
            }
            deferred.resolve(trialJob);
        } catch (error) {
            this.log.error(`Update job status exception, error is ${error.message}`);
            if (error instanceof NNIError && error.name === NNIErrorNames.NOT_FOUND) {
                deferred.resolve(trialJob);
            } else {
                trialJob.status = 'UNKNOWN';
                deferred.resolve(trialJob);
            }
        }
        return deferred.promise;
    }

SparkSnail's avatar
SparkSnail committed
697
698
    private getRemoteScriptsPath(userName: string): string {
        return path.join(getRemoteTmpDir(this.remoteOS), userName, 'nni', 'scripts');
Deshui Yu's avatar
Deshui Yu committed
699
700
701
702
703
704
    }

    private getHostJobRemoteDir(jobId: string): string {
        return path.join(this.remoteExpRootDir, 'hostjobs', jobId);
    }

705
    private getRemoteExperimentRootDir(): string{
706
        return path.join(getRemoteTmpDir(this.remoteOS), 'nni', 'experiments', getExperimentId());
Deshui Yu's avatar
Deshui Yu committed
707
708
    }

SparkSnail's avatar
SparkSnail committed
709
710
711
712
    public get MetricsEmitter() : EventEmitter {
        return this.metricsEmitter;
    }

Deshui Yu's avatar
Deshui Yu committed
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
    private getJobPidPath(jobId: string): string {
        const trialJobDetail: RemoteMachineTrialJobDetail | undefined = this.trialJobsMap.get(jobId);
        if (trialJobDetail === undefined) {
            throw new NNIError(NNIErrorNames.INVALID_JOB_DETAIL, `Invalid job detail information for trial job ${jobId}`);
        }

        let jobpidPath: string;
        if (trialJobDetail.form.jobType === 'TRIAL') {
            jobpidPath = path.join(trialJobDetail.workingDirectory, '.nni', 'jobpid');
        } else if (trialJobDetail.form.jobType === 'HOST') {
            jobpidPath = path.join(this.getHostJobRemoteDir(jobId), 'jobpid');
        } else {
            throw new Error(`Job type not supported: ${trialJobDetail.form.jobType}`);
        }

        return jobpidPath;
    }
chicm-ms's avatar
chicm-ms committed
730
731

    private async writeParameterFile(trialJobId: string, hyperParameters: HyperParameters, rmMeta: RemoteMachineMeta): Promise<void> {
SparkSnail's avatar
SparkSnail committed
732
        const sshClient: Client | undefined = this.trialSSHClientMap.get(trialJobId);
chicm-ms's avatar
chicm-ms committed
733
734
735
736
737
738
739
        if (sshClient === undefined) {
            throw new Error('sshClient is undefined.');
        }

        const trialWorkingFolder: string = path.join(this.remoteExpRootDir, 'trials', trialJobId);
        const trialLocalTempFolder: string = path.join(this.expRootDir, 'trials-local', trialJobId);

740
        const fileName: string = generateParamFileName(hyperParameters);
chicm-ms's avatar
chicm-ms committed
741
742
743
744
745
        const localFilepath: string = path.join(trialLocalTempFolder, fileName);
        await fs.promises.writeFile(localFilepath, hyperParameters.value, { encoding: 'utf8' });

        await SSHClientUtility.copyFileToRemote(localFilepath, path.join(trialWorkingFolder, fileName), sshClient);
    }
746
747

    private generateSequenceId(): number {
748
749
750
751
        if (this.trialSequenceId === -1) {
            this.trialSequenceId = getInitTrialSequenceId();
        }

752
753
        return this.trialSequenceId++;
    }
Deshui Yu's avatar
Deshui Yu committed
754
755
756
}

export { RemoteMachineTrainingService };