"testing/vscode:/vscode.git/clone" did not exist on "2957afcabf7ca5aaa3da2f7c9027a6199b5ba1cb"
Unverified Commit 5fc73ba6 authored by liuzhe-lz's avatar liuzhe-lz Committed by GitHub
Browse files

Refactor NNI Manager UT (Stage 1) - setup toolchain (#4109)

parent 2b9f5f8c
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import assert from 'assert';
import { Deferred } from 'ts-deferred';
import * as component from '../common/component';
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import * as assert from 'assert';
import fs from 'fs';
import os from 'os';
import path from 'path';
import assert from 'assert';
import { getLogger, Logger } from '../common/log';
import { isAlive, withLockSync, getExperimentsInfoPath, delay } from '../common/utils';
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as fs from 'fs';
import * as cp from 'child_process';
import * as path from 'path';
import fs from 'fs';
import cp from 'child_process';
import path from 'path';
import { ChildProcess } from 'child_process';
import * as component from '../common/component';
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import assert from 'assert';
import { ChildProcess, StdioOptions } from 'child_process';
import { Deferred } from 'ts-deferred';
import * as component from '../common/component';
......@@ -470,8 +468,6 @@ class NNIManager implements Manager {
const module_ = await import('../training_service/reusable/routerTrainingService');
return await module_.RouterTrainingService.construct(config);
}
throw new Error(`Unsupported training service platform "${platform}"`);
}
private setupTuner(command: string, cwd: string | undefined, mode: 'start' | 'resume', dataDirectory: string): void {
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import * as fs from 'fs';
import * as path from 'path';
import * as sqlite3 from 'sqlite3';
import assert from 'assert';
import fs from 'fs';
import path from 'path';
import sqlite3 from 'sqlite3';
import { Deferred } from 'ts-deferred';
import {
......@@ -202,7 +200,7 @@ class SqlDB implements Database {
return deferred.promise;
}
public storeMetricData(trialJobId: string, data: string): Promise<void> {
public storeMetricData(_trialJobId: string, data: string): Promise<void> {
const sql: string = 'insert into MetricData values (?,?,?,?,?,?)';
const json: MetricDataRecord = JSON.parse(data);
const args: any[] = [Date.now(), json.trialJobId, json.parameterId, json.type, json.sequence, JSON.stringify(json.data)];
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import 'app-module-path/register';
import { Container, Scope } from 'typescript-ioc';
import * as fs from 'fs';
......@@ -30,7 +29,7 @@ function initStartupInfo(
setExperimentStartupInfo(createNew, experimentId, basePort, platform, logDirectory, experimentLogLevel, readonly, dispatcherPipe, urlprefix);
}
async function initContainer(foreground: boolean, platformMode: string, logFileName?: string): Promise<void> {
async function initContainer(foreground: boolean, _platformMode: string, logFileName?: string): Promise<void> {
Container.bind(Manager)
.to(NNIManager)
.scope(Scope.Singleton);
......
......@@ -4,13 +4,14 @@
"main": "index.js",
"scripts": {
"build": "tsc",
"test": "nyc --reporter=cobertura --reporter=html --reporter=text --report-dir=./htmlcov mocha -r ts-node/register -t 15000 --recursive test/**/*.test.ts --colors",
"test": "nyc --reporter=cobertura --reporter=text mocha test/**/*.test.ts",
"start": "node dist/main.js",
"watch": "tsc --watch",
"eslint": "npx eslint ./ --ext .ts"
},
"license": "MIT",
"dependencies": {
"app-module-path": "^2.2.0",
"azure-storage": "^2.10.4",
"child-process-promise": "^2.2.1",
"express": "^4.17.1",
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as bodyParser from 'body-parser';
import * as express from 'express';
import * as httpProxy from 'http-proxy';
import * as path from 'path';
import bodyParser from 'body-parser';
import express from 'express';
import httpProxy from 'http-proxy';
import path from 'path';
import * as component from '../common/component';
import { RestServer } from '../common/restServer'
import { getLogDir } from '../common/utils';
......@@ -50,7 +48,7 @@ export class NNIRestServer extends RestServer {
target: 'https://netron.app'
});
});
this.app.get(`${getPrefixUrl()}/*`, (req: express.Request, res: express.Response) => {
this.app.get(`${getPrefixUrl()}/*`, (_req: express.Request, res: express.Response) => {
res.sendFile(path.resolve('static/index.html'));
});
}
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { Request, Response, Router } from 'express';
import * as path from 'path';
import path from 'path';
import * as component from '../common/component';
import { DataStore, MetricDataRecord, TrialJobInfo } from '../common/datastore';
......@@ -80,7 +78,7 @@ class NNIRestHandler {
this.stop(router);
// Express-joi-validator configuration
router.use((err: any, _req: Request, res: Response, _next: any) => {
router.use((err: any, _req: Request, res: Response, _next: any): any => {
if (err.isBoom) {
this.log.error(err.output.payload);
......@@ -111,7 +109,7 @@ class NNIRestHandler {
}
private version(router: Router): void {
router.get('/version', async (req: Request, res: Response) => {
router.get('/version', async (_req: Request, res: Response) => {
const version = await getVersion();
res.send(version);
});
......@@ -119,7 +117,7 @@ class NNIRestHandler {
// TODO add validators for request params, query, body
private checkStatus(router: Router): void {
router.get('/check-status', (req: Request, res: Response) => {
router.get('/check-status', (_req: Request, res: Response) => {
const ds: DataStore = component.get<DataStore>(DataStore);
ds.init().then(() => {
res.send(this.nniManager.getStatus());
......@@ -133,7 +131,7 @@ class NNIRestHandler {
}
private getExperimentProfile(router: Router): void {
router.get('/experiment', (req: Request, res: Response) => {
router.get('/experiment', (_req: Request, res: Response) => {
this.nniManager.getExperimentProfile().then((profile: ExperimentProfile) => {
res.send(profile);
}).catch((err: Error) => {
......@@ -144,7 +142,7 @@ class NNIRestHandler {
private updateExperimentProfile(router: Router): void {
router.put('/experiment', (req: Request, res: Response) => {
this.nniManager.updateExperimentProfile(req.body, req.query.update_type as ProfileUpdateType).then(() => {
this.nniManager.updateExperimentProfile(req.body, req.query['update_type'] as ProfileUpdateType).then(() => {
res.send();
}).catch((err: Error) => {
this.handleError(err, res);
......@@ -163,7 +161,7 @@ class NNIRestHandler {
}
private getImportedData(router: Router): void {
router.get('/experiment/imported-data', (req: Request, res: Response) => {
router.get('/experiment/imported-data', (_req: Request, res: Response) => {
this.nniManager.getImportedData().then((importedData: string[]) => {
res.send(JSON.stringify(importedData));
}).catch((err: Error) => {
......@@ -195,7 +193,7 @@ class NNIRestHandler {
}
private getTrialJobStatistics(router: Router): void {
router.get('/job-statistics', (req: Request, res: Response) => {
router.get('/job-statistics', (_req: Request, res: Response) => {
this.nniManager.getTrialJobStatistics().then((statistics: TrialJobStatistics[]) => {
res.send(statistics);
}).catch((err: Error) => {
......@@ -224,7 +222,7 @@ class NNIRestHandler {
private listTrialJobs(router: Router): void {
router.get('/trial-jobs', (req: Request, res: Response) => {
this.nniManager.listTrialJobs(req.query.status as TrialJobStatus).then((jobInfos: TrialJobInfo[]) => {
this.nniManager.listTrialJobs(req.query['status'] as TrialJobStatus).then((jobInfos: TrialJobInfo[]) => {
jobInfos.forEach((trialJob: TrialJobInfo) => {
this.setErrorPathForFailedJob(trialJob);
});
......@@ -237,7 +235,7 @@ class NNIRestHandler {
private getTrialJob(router: Router): void {
router.get('/trial-jobs/:id', (req: Request, res: Response) => {
this.nniManager.getTrialJob(req.params.id).then((jobDetail: TrialJobInfo) => {
this.nniManager.getTrialJob(req.params['id']).then((jobDetail: TrialJobInfo) => {
const jobInfo: TrialJobInfo = this.setErrorPathForFailedJob(jobDetail);
res.send(jobInfo);
}).catch((err: Error) => {
......@@ -258,7 +256,7 @@ class NNIRestHandler {
private cancelTrialJob(router: Router): void {
router.delete('/trial-jobs/:id', async (req: Request, res: Response) => {
this.nniManager.cancelTrialJobByUser(req.params.id).then(() => {
this.nniManager.cancelTrialJobByUser(req.params['id']).then(() => {
res.send();
}).catch((err: Error) => {
this.handleError(err, res);
......@@ -268,7 +266,7 @@ class NNIRestHandler {
private getMetricData(router: Router): void {
router.get('/metric-data/:job_id*?', async (req: Request, res: Response) => {
this.nniManager.getMetricData(req.params.job_id, req.query.type as MetricType).then((metricsData: MetricDataRecord[]) => {
this.nniManager.getMetricData(req.params['job_id'], req.query['type'] as MetricType).then((metricsData: MetricDataRecord[]) => {
res.send(metricsData);
}).catch((err: Error) => {
this.handleError(err, res);
......@@ -278,8 +276,8 @@ class NNIRestHandler {
private getMetricDataByRange(router: Router): void {
router.get('/metric-data-range/:min_seq_id/:max_seq_id', async (req: Request, res: Response) => {
const minSeqId = Number(req.params.min_seq_id);
const maxSeqId = Number(req.params.max_seq_id);
const minSeqId = Number(req.params['min_seq_id']);
const maxSeqId = Number(req.params['max_seq_id']);
this.nniManager.getMetricDataByRange(minSeqId, maxSeqId).then((metricsData: MetricDataRecord[]) => {
res.send(metricsData);
}).catch((err: Error) => {
......@@ -289,7 +287,7 @@ class NNIRestHandler {
}
private getLatestMetricData(router: Router): void {
router.get('/metric-data-latest/', async (req: Request, res: Response) => {
router.get('/metric-data-latest/', async (_req: Request, res: Response) => {
this.nniManager.getLatestMetricData().then((metricsData: MetricDataRecord[]) => {
res.send(metricsData);
}).catch((err: Error) => {
......@@ -301,11 +299,11 @@ class NNIRestHandler {
private getTrialFile(router: Router): void {
router.get('/trial-file/:id/:filename', async(req: Request, res: Response) => {
let encoding: string | null = null;
const filename = req.params.filename;
const filename = req.params['filename'];
if (!filename.includes('.') || filename.match(/.*\.(txt|log)/g)) {
encoding = 'utf8';
}
this.nniManager.getTrialFile(req.params.id, filename).then((content: Buffer | string) => {
this.nniManager.getTrialFile(req.params['id'], filename).then((content: Buffer | string) => {
const contentType = content instanceof Buffer ? 'application/octet-stream' : 'text/plain';
res.header('Content-Type', contentType);
if (content === '') {
......@@ -319,7 +317,7 @@ class NNIRestHandler {
}
private exportData(router: Router): void {
router.get('/export-data', (req: Request, res: Response) => {
router.get('/export-data', (_req: Request, res: Response) => {
this.nniManager.exportData().then((exportedData: string) => {
res.send(exportedData);
}).catch((err: Error) => {
......@@ -329,7 +327,7 @@ class NNIRestHandler {
}
private getExperimentMetadata(router: Router): void {
router.get('/experiment-metadata', (req: Request, res: Response) => {
router.get('/experiment-metadata', (_req: Request, res: Response) => {
Promise.all([
this.nniManager.getExperimentProfile(),
this.experimentsManager.getExperimentsInfo()
......@@ -347,7 +345,7 @@ class NNIRestHandler {
}
private getExperimentsInfo(router: Router): void {
router.get('/experiments-info', (req: Request, res: Response) => {
router.get('/experiments-info', (_req: Request, res: Response) => {
this.experimentsManager.getExperimentsInfo().then((experimentInfo: JSON) => {
res.send(JSON.stringify(experimentInfo));
}).catch((err: Error) => {
......@@ -369,7 +367,7 @@ class NNIRestHandler {
private getTensorboardTask(router: Router): void {
router.get('/tensorboard/:id', (req: Request, res: Response) => {
this.tensorboardManager.getTensorboardTask(req.params.id).then((taskDetail: TensorboardTaskInfo) => {
this.tensorboardManager.getTensorboardTask(req.params['id']).then((taskDetail: TensorboardTaskInfo) => {
res.send(Object.assign({}, taskDetail));
}).catch((err: Error) => {
this.handleError(err, res);
......@@ -379,7 +377,7 @@ class NNIRestHandler {
private updateTensorboardTask(router: Router): void {
router.put('/tensorboard/:id', (req: Request, res: Response) => {
this.tensorboardManager.updateTensorboardTask(req.params.id).then((taskDetail: TensorboardTaskInfo) => {
this.tensorboardManager.updateTensorboardTask(req.params['id']).then((taskDetail: TensorboardTaskInfo) => {
res.send(Object.assign({}, taskDetail));
}).catch((err: Error) => {
this.handleError(err, res);
......@@ -389,7 +387,7 @@ class NNIRestHandler {
private stopTensorboardTask(router: Router): void {
router.delete('/tensorboard/:id', (req: Request, res: Response) => {
this.tensorboardManager.stopTensorboardTask(req.params.id).then((taskDetail: TensorboardTaskInfo) => {
this.tensorboardManager.stopTensorboardTask(req.params['id']).then((taskDetail: TensorboardTaskInfo) => {
res.send(Object.assign({}, taskDetail));
}).catch((err: Error) => {
this.handleError(err, res);
......@@ -398,7 +396,7 @@ class NNIRestHandler {
}
private stopAllTensorboardTask(router: Router): void {
router.delete('/tensorboard-tasks', (req: Request, res: Response) => {
router.delete('/tensorboard-tasks', (_req: Request, res: Response) => {
this.tensorboardManager.stopAllTensorboardTask().then(() => {
res.send();
}).catch((err: Error) => {
......@@ -408,7 +406,7 @@ class NNIRestHandler {
}
private listTensorboardTask(router: Router): void {
router.get('/tensorboard-tasks', (req: Request, res: Response) => {
router.get('/tensorboard-tasks', (_req: Request, res: Response) => {
this.tensorboardManager.listTensorboardTasks().then((taskDetails: TensorboardTaskInfo[]) => {
res.send(taskDetails);
}).catch((err: Error) => {
......@@ -418,7 +416,7 @@ class NNIRestHandler {
}
private stop(router: Router): void {
router.delete('/experiment', (req: Request, res: Response) => {
router.delete('/experiment', (_req: Request, res: Response) => {
this.nniManager.stopExperimentTopHalf().then(() => {
res.send();
this.nniManager.stopExperimentBottomHalf();
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
const joi = require('joi');
export namespace ValidationSchemas {
......
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
import * as assert from 'assert';
import assert from 'assert';
import { getIPV4Address } from '../../common/utils';
it('getIpv4Address', async () => {
......
......@@ -43,7 +43,7 @@ function startProcess(): void {
}
);
const proc: ChildProcess = getTunerProc(dispatcherCmd, stdio, 'core/test', process.env);
proc.on('error', (error: Error): void => {
proc.on('error', (_error: Error): void => {
procExit = true;
procError = true;
});
......
......@@ -169,9 +169,9 @@ describe('Unit test for nnimanager', function () {
it('test getTrialJob with invalid id', () => {
//query a not exist id, and the function should throw error, and should not process then() method
return nniManager.getTrialJob('4567').then((jobid) => {
return nniManager.getTrialJob('4567').then((_jobid) => {
assert.fail();
}).catch((error) => {
}).catch((_error) => {
assert.isTrue(true);
})
})
......@@ -252,9 +252,9 @@ describe('Unit test for nnimanager', function () {
it('test getMetricData with invalid trialJobId', () => {
//query an invalid trialJobId
return nniManager.getMetricData('43210', 'CUSTOM').then((metricData) => {
return nniManager.getMetricData('43210', 'CUSTOM').then((_metricData) => {
assert.fail();
}).catch((error) => {
}).catch((_error) => {
})
})
......
......@@ -204,7 +204,7 @@ class MockedDataStore implements DataStore {
return Promise.resolve(ret);
}
public getTrialJob(trialJobId: string): Promise<TrialJobInfo> {
public getTrialJob(_trialJobId: string): Promise<TrialJobInfo> {
return Promise.resolve({
trialJobId: '1234',
status: 'SUCCEEDED',
......
......@@ -25,16 +25,16 @@ export class mockedeExperimentManager extends ExperimentManager {
"logDir": null
}
}));
return new Promise<JSON>((resolve, reject) => {
return new Promise<JSON>((resolve, _reject) => {
resolve(expInfo);
});
}
public setExperimentPath(newPath: string): void {
public setExperimentPath(_newPath: string): void {
return
}
public setExperimentInfo(experimentId: string, key: string, value: any): void {
public setExperimentInfo(_experimentId: string, _key: string, _value: any): void {
return
}
......
......@@ -27,10 +27,10 @@ export class MockedNNIManager extends Manager {
errors: []
}
}
public updateExperimentProfile(experimentProfile: ExperimentProfile, updateType: ProfileUpdateType): Promise<void> {
public updateExperimentProfile(_experimentProfile: ExperimentProfile, _updateType: ProfileUpdateType): Promise<void> {
return Promise.resolve();
}
public importData(data: string): Promise<void> {
public importData(_data: string): Promise<void> {
return Promise.resolve();
}
public getImportedData(): Promise<string[]> {
......@@ -53,7 +53,7 @@ export class MockedNNIManager extends Manager {
return deferred.promise;
}
public addCustomizedTrialJob(hyperParams: string): Promise<number> {
public addCustomizedTrialJob(_hyperParams: string): Promise<number> {
return Promise.resolve(99);
}
......@@ -61,7 +61,7 @@ export class MockedNNIManager extends Manager {
return Promise.resolve();
}
public submitTrialJob(form: TrialJobApplicationForm): Promise<TrialJobDetail> {
public submitTrialJob(_form: TrialJobApplicationForm): Promise<TrialJobDetail> {
const deferred: Deferred<TrialJobDetail> = new Deferred<TrialJobDetail>();
const jobDetail: TrialJobDetail = {
id: '1234',
......@@ -82,23 +82,23 @@ export class MockedNNIManager extends Manager {
return deferred.promise;
}
public cancelTrialJobByUser(trialJobId: string): Promise<void> {
public cancelTrialJobByUser(_trialJobId: string): Promise<void> {
return Promise.resolve();
}
public getClusterMetadata(key: string): Promise<string> {
public getClusterMetadata(_key: string): Promise<string> {
return Promise.resolve('METAVALUE1');
}
public startExperiment(experimentParams: ExperimentConfig): Promise<string> {
public startExperiment(_experimentParams: ExperimentConfig): Promise<string> {
return Promise.resolve('id-1234');
}
public setClusterMetadata(key: string, value: string): Promise<void> {
public setClusterMetadata(_key: string, _value: string): Promise<void> {
return Promise.resolve();
}
public getTrialJob(trialJobId: string): Promise<TrialJobInfo> {
public getTrialJob(_trialJobId: string): Promise<TrialJobInfo> {
const deferred: Deferred<TrialJobInfo> = new Deferred<TrialJobInfo>();
const jobInfo: TrialJobInfo = {
trialJobId: '1234',
......@@ -120,16 +120,16 @@ export class MockedNNIManager extends Manager {
public stopExperimentBottomHalf(): Promise<void> {
throw new MethodNotImplementedError();
}
public getMetricData(trialJobId: string, metricType: MetricType): Promise<MetricDataRecord[]> {
public getMetricData(_trialJobId: string, _metricType: MetricType): Promise<MetricDataRecord[]> {
throw new MethodNotImplementedError();
}
public getMetricDataByRange(minSeqId: number, maxSeqId: number): Promise<MetricDataRecord[]> {
public getMetricDataByRange(_minSeqId: number, _maxSeqId: number): Promise<MetricDataRecord[]> {
throw new MethodNotImplementedError();
}
public getLatestMetricData(): Promise<MetricDataRecord[]> {
throw new MethodNotImplementedError();
}
public getTrialFile(trialJobId: string, fileName: string): Promise<string> {
public getTrialFile(_trialJobId: string, _fileName: string): Promise<string> {
throw new MethodNotImplementedError();
}
public getExperimentProfile(): Promise<ExperimentProfile> {
......@@ -161,7 +161,7 @@ export class MockedNNIManager extends Manager {
return Promise.resolve(profile);
}
public listTrialJobs(status?: TrialJobStatus): Promise<TrialJobInfo[]> {
public listTrialJobs(_status?: TrialJobStatus): Promise<TrialJobInfo[]> {
const job1: TrialJobInfo = {
trialJobId: '1234',
status: 'SUCCEEDED',
......
......@@ -63,7 +63,7 @@ class MockedTrainingService extends TrainingService {
return deferred.promise;
}
public getTrialFile(trialJobId: string, fileName: string): Promise<string> {
public getTrialFile(_trialJobId: string, _fileName: string): Promise<string> {
throw new MethodNotImplementedError();
}
......@@ -71,18 +71,18 @@ class MockedTrainingService extends TrainingService {
}
public addTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void {
public addTrialJobMetricListener(_listener: (_metric: TrialJobMetric) => void): void {
}
public removeTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void {
public removeTrialJobMetricListener(_listener: (_metric: TrialJobMetric) => void): void {
}
public submitTrialJob(form: TrialJobApplicationForm): Promise<TrialJobDetail> {
public submitTrialJob(_form: TrialJobApplicationForm): Promise<TrialJobDetail> {
const deferred = new Deferred<TrialJobDetail>();
return deferred.promise;
}
public updateTrialJob(trialJobId: string, form: TrialJobApplicationForm): Promise<TrialJobDetail> {
public updateTrialJob(_trialJobId: string, _form: TrialJobApplicationForm): Promise<TrialJobDetail> {
throw new MethodNotImplementedError();
}
......@@ -90,7 +90,7 @@ class MockedTrainingService extends TrainingService {
return false;
}
public cancelTrialJob(trialJobId: string, isEarlyStopped: boolean = false): Promise<void> {
public cancelTrialJob(trialJobId: string, _isEarlyStopped: boolean = false): Promise<void> {
const deferred = new Deferred<void>();
if(trialJobId === '1234' || trialJobId === '3456'){
deferred.resolve();
......
require('ts-node/register');
require('app-module-path/cwd');
......@@ -4,7 +4,7 @@
'use strict';
import { assert, expect } from 'chai';
import * as request from 'request';
import request from 'request';
import { Container } from 'typescript-ioc';
import * as component from '../../common/component';
......
......@@ -3,10 +3,10 @@
'use strict';
import * as chai from 'chai';
import * as chaiAsPromised from 'chai-as-promised';
import * as fs from 'fs';
import * as tmp from 'tmp';
import chai from 'chai';
import chaiAsPromised from 'chai-as-promised';
import fs from 'fs';
import tmp from 'tmp';
import * as component from '../../common/component';
import { TrialJobApplicationForm, TrialJobDetail, TrainingService } from '../../common/trainingService';
import { cleanupUnitTest, prepareUnitTest } from '../../common/utils';
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment