localTrainingService.test.ts 5.24 KB
Newer Older
liuzhe-lz's avatar
liuzhe-lz committed
1
2
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
Deshui Yu's avatar
Deshui Yu committed
3
4
5

'use strict';

6
7
8
9
10
import chai from 'chai';
import chaiAsPromised from 'chai-as-promised';
import fs from 'fs';
import path from 'path';
import tmp from 'tmp';
Deshui Yu's avatar
Deshui Yu committed
11
import * as component from '../../common/component';
12
13
import { TrialJobApplicationForm, TrialJobDetail} from '../../common/trainingService';
import { cleanupUnitTest, delay, prepareUnitTest, getExperimentRootDir } from '../../common/utils';
liuzhe-lz's avatar
liuzhe-lz committed
14
15
import { TrialConfigMetadataKey } from '../../training_service/common/trialConfigMetadataKey';
import { LocalTrainingService } from '../../training_service/local/localTrainingService';
SparkSnail's avatar
SparkSnail committed
16
17

// TODO: copy mockedTrail.py to local folder
18
const localCodeDir: string = tmp.dirSync().name.split('\\').join('\\\\');
liuzhe-lz's avatar
liuzhe-lz committed
19
const mockedTrialPath: string = './test/mock/mockedTrial.py'
SparkSnail's avatar
SparkSnail committed
20
fs.copyFileSync(mockedTrialPath, localCodeDir + '/mockedTrial.py')
Deshui Yu's avatar
Deshui Yu committed
21
22

describe('Unit Test for LocalTrainingService', () => {
liuzhe-lz's avatar
liuzhe-lz committed
23
24
    const config = <any>{
        platform: 'local',
25
26
        trialCommand: 'sleep 1h && echo hello',
        trialCodeDirectory: `${localCodeDir}`,
liuzhe-lz's avatar
liuzhe-lz committed
27
        trialGpuNumber: 0,  // TODO: add test case for gpu?
liuzhe-lz's avatar
liuzhe-lz committed
28
29
        maxTrialNumberPerGpu: 1,
        reuseMode: true,
30
31
    };

liuzhe-lz's avatar
liuzhe-lz committed
32
33
    const config2 = <any>{
        platform: 'local',
34
35
36
        trialCommand: 'python3 mockedTrial.py',
        trialCodeDirectory: `${localCodeDir}`,
        trialGpuNumber: 0,
liuzhe-lz's avatar
liuzhe-lz committed
37
38
        maxTrialNumberPerGpu: 1,
        reuseMode: true,
39
    };
SparkSnail's avatar
SparkSnail committed
40
41
42
43
44
45
46
47
48
49
50

    before(() => {
        chai.should();
        chai.use(chaiAsPromised);
        prepareUnitTest();
    });

    after(() => {
        cleanupUnitTest();
    });

51
52
    it('List empty trial jobs', async () => {
        const localTrainingService = new LocalTrainingService(config);
SparkSnail's avatar
SparkSnail committed
53
54
55
56
        localTrainingService.run();

        //trial jobs should be empty, since there are no submitted jobs
        chai.expect(await localTrainingService.listTrialJobs()).to.be.empty;
57

58
        localTrainingService.cleanUp();
SparkSnail's avatar
SparkSnail committed
59
60
61
    });

    it('Submit job and Cancel job', async () => {
62
63
        const localTrainingService = new LocalTrainingService(config);
        localTrainingService.run();
SparkSnail's avatar
SparkSnail committed
64
65
66

        // submit job
        const form: TrialJobApplicationForm = {
67
            sequenceId: 0,
SparkSnail's avatar
SparkSnail committed
68
69
70
71
72
73
74
75
76
            hyperParameters: {
                value: 'mock hyperparameters',
                index: 0
            }
        };
        const jobDetail: TrialJobDetail = await localTrainingService.submitTrialJob(form);
        chai.expect(jobDetail.status).to.be.equals('WAITING');
        await localTrainingService.cancelTrialJob(jobDetail.id);
        chai.expect(jobDetail.status).to.be.equals('USER_CANCELED');
77
78

        localTrainingService.cleanUp();
SparkSnail's avatar
SparkSnail committed
79
    }).timeout(20000);
80

81
    it('Get trial log', async () => {
82
83
        const localTrainingService = new LocalTrainingService(config);
        localTrainingService.run();
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101

        // submit job
        const form: TrialJobApplicationForm = {
            sequenceId: 0,
            hyperParameters: {
                value: 'mock hyperparameters',
                index: 0
            }
        };

        const jobDetail: TrialJobDetail = await localTrainingService.submitTrialJob(form);

        // get trial log
        const rootDir: string = getExperimentRootDir()
        fs.mkdirSync(path.join(rootDir, 'trials'))
        fs.mkdirSync(jobDetail.workingDirectory)
        fs.writeFileSync(path.join(jobDetail.workingDirectory, 'trial.log'), 'trial log')
        fs.writeFileSync(path.join(jobDetail.workingDirectory, 'stderr'), 'trial stderr')
Yuge Zhang's avatar
Yuge Zhang committed
102
103
        chai.expect(await localTrainingService.getTrialFile(jobDetail.id, 'trial.log')).to.be.equals('trial log');
        chai.expect(await localTrainingService.getTrialFile(jobDetail.id, 'stderr')).to.be.equals('trial stderr');
104
105
106
107
108
109
        fs.unlinkSync(path.join(jobDetail.workingDirectory, 'trial.log'))
        fs.unlinkSync(path.join(jobDetail.workingDirectory, 'stderr'))
        fs.rmdirSync(jobDetail.workingDirectory)
        fs.rmdirSync(path.join(rootDir, 'trials'))

        await localTrainingService.cancelTrialJob(jobDetail.id);
110
        localTrainingService.cleanUp();
111
112
    }).timeout(20000);

SparkSnail's avatar
SparkSnail committed
113
    it('Read metrics, Add listener, and remove listener', async () => {
114
115
        const localTrainingService = new LocalTrainingService(config2);
        localTrainingService.run();
SparkSnail's avatar
SparkSnail committed
116

117
        // set meta data
SparkSnail's avatar
SparkSnail committed
118
119
        // submit job
        const form: TrialJobApplicationForm = {
120
            sequenceId: 0,
SparkSnail's avatar
SparkSnail committed
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
            hyperParameters: {
                value: 'mock hyperparameters',
                index: 0
            }
        };
        const jobDetail: TrialJobDetail = await localTrainingService.submitTrialJob(form);
        chai.expect(jobDetail.status).to.be.equals('WAITING');
        localTrainingService.listTrialJobs().then((jobList)=>{
            chai.expect(jobList.length).to.be.equals(1);
        });
        // Add metrics listeners
        const listener1 = function f1(metric: any) {
            chai.expect(metric.id).to.be.equals(jobDetail.id);
        }
        localTrainingService.addTrialJobMetricListener(listener1);
        // Wait to collect metric
        await delay(1000);

        await localTrainingService.cancelTrialJob(jobDetail.id);
        localTrainingService.removeTrialJobMetricListener(listener1);
141
        localTrainingService.cleanUp();
SparkSnail's avatar
SparkSnail committed
142
    }).timeout(20000);
143
});