"git@developer.sourcefind.cn:OpenDAS/nni.git" did not exist on "496f653bc1129c7f3c225cd8e4c81900135c9001"
Unverified Commit 6485ef17 authored by Chi Song's avatar Chi Song Committed by GitHub
Browse files

refactor: update eslint rules to remove warnings during build (#2405)

update nni manager eslint rules.

1. add rule to ignore unused args, and treat other unused as error.
2. add run to treat no explicit function return type as error.
3. fix all errors.
parent f67cebde
...@@ -23,6 +23,13 @@ ...@@ -23,6 +23,13 @@
"@typescript-eslint/consistent-type-assertions": 0, "@typescript-eslint/consistent-type-assertions": 0,
"@typescript-eslint/no-inferrable-types": 0, "@typescript-eslint/no-inferrable-types": 0,
"no-inner-declarations": 0, "no-inner-declarations": 0,
"@typescript-eslint/explicit-function-return-type": "error",
"@typescript-eslint/no-unused-vars": [
"error",
{
"argsIgnorePattern": "^_"
}
],
"@typescript-eslint/no-var-requires": 0 "@typescript-eslint/no-var-requires": 0
}, },
"ignorePatterns": [ "ignorePatterns": [
......
...@@ -98,7 +98,7 @@ class SqlDB implements Database { ...@@ -98,7 +98,7 @@ class SqlDB implements Database {
this.resolve(this.initTask, err); this.resolve(this.initTask, err);
} else { } else {
if (createNew) { if (createNew) {
this.db.exec(createTables, (error: Error | null) => { this.db.exec(createTables, (_error: Error | null) => {
this.resolve(this.initTask, err); this.resolve(this.initTask, err);
}); });
} else { } else {
......
...@@ -60,7 +60,7 @@ class NNIRestHandler { ...@@ -60,7 +60,7 @@ class NNIRestHandler {
this.exportData(router); this.exportData(router);
// Express-joi-validator configuration // Express-joi-validator configuration
router.use((err: any, req: Request, res: Response, next: any) => { router.use((err: any, _req: Request, res: Response, _next: any) => {
if (err.isBoom) { if (err.isBoom) {
this.log.error(err.output.payload); this.log.error(err.output.payload);
......
...@@ -131,7 +131,7 @@ class DLTSTrainingService implements TrainingService { ...@@ -131,7 +131,7 @@ class DLTSTrainingService implements TrainingService {
private async statusCheckingLoop(): Promise<void> { private async statusCheckingLoop(): Promise<void> {
while (!this.stopping) { while (!this.stopping) {
const updateDLTSTrialJobs: Promise<void>[] = []; const updateDLTSTrialJobs: Promise<void>[] = [];
for (const [trialJobId, dltsTrialJob] of this.trialJobsMap) { for (const dltsTrialJob of this.trialJobsMap.values()) {
updateDLTSTrialJobs.push(this.getDLTSTrialJobInfo(dltsTrialJob)); updateDLTSTrialJobs.push(this.getDLTSTrialJobInfo(dltsTrialJob));
} }
...@@ -405,7 +405,7 @@ class DLTSTrainingService implements TrainingService { ...@@ -405,7 +405,7 @@ class DLTSTrainingService implements TrainingService {
} }
} }
public async getClusterMetadata(key: string): Promise<string> { public async getClusterMetadata(_key: string): Promise<string> {
return ''; return '';
} }
...@@ -545,7 +545,7 @@ class DLTSTrainingService implements TrainingService { ...@@ -545,7 +545,7 @@ class DLTSTrainingService implements TrainingService {
body: parameterFileMeta body: parameterFileMeta
}; };
await new Promise((resolve, reject) => { await new Promise((resolve, reject) => {
request(req, (err: Error, res: request.Response) => { request(req, (err: Error, _res: request.Response) => {
if (err) { if (err) {
reject(err); reject(err);
} else { } else {
......
...@@ -20,7 +20,7 @@ export namespace AzureStorageClientUtility { ...@@ -20,7 +20,7 @@ export namespace AzureStorageClientUtility {
*/ */
export async function createShare(fileServerClient: any, azureShare: any): Promise<boolean> { export async function createShare(fileServerClient: any, azureShare: any): Promise<boolean> {
const deferred: Deferred<boolean> = new Deferred<boolean>(); const deferred: Deferred<boolean> = new Deferred<boolean>();
fileServerClient.createShareIfNotExists(azureShare, (error: any, result: any, response: any) => { fileServerClient.createShareIfNotExists(azureShare, (error: any, _result: any, _response: any) => {
if (error) { if (error) {
getLogger() getLogger()
.error(`Create share failed:, ${error}`); .error(`Create share failed:, ${error}`);
...@@ -41,7 +41,7 @@ export namespace AzureStorageClientUtility { ...@@ -41,7 +41,7 @@ export namespace AzureStorageClientUtility {
*/ */
export async function createDirectory(fileServerClient: azureStorage.FileService, azureFoler: any, azureShare: any): Promise<boolean> { export async function createDirectory(fileServerClient: azureStorage.FileService, azureFoler: any, azureShare: any): Promise<boolean> {
const deferred: Deferred<boolean> = new Deferred<boolean>(); const deferred: Deferred<boolean> = new Deferred<boolean>();
fileServerClient.createDirectoryIfNotExists(azureShare, azureFoler, (error: any, result: any, response: any) => { fileServerClient.createDirectoryIfNotExists(azureShare, azureFoler, (error: any, _result: any, _response: any) => {
if (error) { if (error) {
getLogger() getLogger()
.error(`Create directory failed:, ${error}`); .error(`Create directory failed:, ${error}`);
...@@ -89,7 +89,7 @@ export namespace AzureStorageClientUtility { ...@@ -89,7 +89,7 @@ export namespace AzureStorageClientUtility {
localFilePath: string): Promise<boolean> { localFilePath: string): Promise<boolean> {
const deferred: Deferred<boolean> = new Deferred<boolean>(); const deferred: Deferred<boolean> = new Deferred<boolean>();
await fileServerClient.createFileFromLocalFile(azureShare, azureDirectory, azureFileName, localFilePath, await fileServerClient.createFileFromLocalFile(azureShare, azureDirectory, azureFileName, localFilePath,
(error: any, result: any, response: any) => { (error: any, _result: any, _response: any) => {
if (error) { if (error) {
getLogger() getLogger()
.error(`Upload file failed:, ${error}`); .error(`Upload file failed:, ${error}`);
...@@ -114,7 +114,7 @@ export namespace AzureStorageClientUtility { ...@@ -114,7 +114,7 @@ export namespace AzureStorageClientUtility {
localFilePath: string): Promise<boolean> { localFilePath: string): Promise<boolean> {
const deferred: Deferred<boolean> = new Deferred<boolean>(); const deferred: Deferred<boolean> = new Deferred<boolean>();
await fileServerClient.getFileToStream(azureShare, azureDirectory, azureFileName, fs.createWriteStream(localFilePath), await fileServerClient.getFileToStream(azureShare, azureDirectory, azureFileName, fs.createWriteStream(localFilePath),
(error: any, result: any, response: any) => { (error: any, _result: any, _response: any) => {
if (error) { if (error) {
getLogger() getLogger()
.error(`Download file failed:, ${error}`); .error(`Download file failed:, ${error}`);
...@@ -183,7 +183,7 @@ export namespace AzureStorageClientUtility { ...@@ -183,7 +183,7 @@ export namespace AzureStorageClientUtility {
const deferred: Deferred<void> = new Deferred<void>(); const deferred: Deferred<void> = new Deferred<void>();
await mkDirP(localDirectory); await mkDirP(localDirectory);
fileServerClient.listFilesAndDirectoriesSegmented(azureShare, azureDirectory, 'null', fileServerClient.listFilesAndDirectoriesSegmented(azureShare, azureDirectory, 'null',
async (error: any, result: any, response: any) => { async (_error: any, result: any, _response: any) => {
if (('entries' in result) === false) { if (('entries' in result) === false) {
getLogger() getLogger()
.error(`list files failed, can't get entries in result`); .error(`list files failed, can't get entries in result`);
......
...@@ -40,8 +40,8 @@ export class KubernetesJobInfoCollector { ...@@ -40,8 +40,8 @@ export class KubernetesJobInfoCollector {
await Promise.all(updateKubernetesTrialJobs); await Promise.all(updateKubernetesTrialJobs);
} }
protected async retrieveSingleTrialJobInfo(kubernetesCRDClient: KubernetesCRDClient | undefined, protected async retrieveSingleTrialJobInfo(_kubernetesCRDClient: KubernetesCRDClient | undefined,
kubernetesTrialJob: KubernetesTrialJobDetail): Promise<void> { _kubernetesTrialJob: KubernetesTrialJobDetail): Promise<void> {
throw new MethodNotImplementedError(); throw new MethodNotImplementedError();
} }
} }
...@@ -77,7 +77,7 @@ abstract class KubernetesTrainingService { ...@@ -77,7 +77,7 @@ abstract class KubernetesTrainingService {
public async listTrialJobs(): Promise<TrialJobDetail[]> { public async listTrialJobs(): Promise<TrialJobDetail[]> {
const jobs: TrialJobDetail[] = []; const jobs: TrialJobDetail[] = [];
for (const [key, value] of this.trialJobsMap) { for (const key of this.trialJobsMap.keys()) {
jobs.push(await this.getTrialJob(key)); jobs.push(await this.getTrialJob(key));
} }
...@@ -107,7 +107,7 @@ abstract class KubernetesTrainingService { ...@@ -107,7 +107,7 @@ abstract class KubernetesTrainingService {
return false; return false;
} }
public getClusterMetadata(key: string): Promise<string> { public getClusterMetadata(_key: string): Promise<string> {
return Promise.resolve(''); return Promise.resolve('');
} }
......
...@@ -3,7 +3,6 @@ ...@@ -3,7 +3,6 @@
'use strict'; 'use strict';
import {TrialConfig} from '../common/trialConfig';
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService'; import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService';
export class PAIClusterConfig { export class PAIClusterConfig {
......
...@@ -62,7 +62,7 @@ export class PAIJobInfoCollector { ...@@ -62,7 +62,7 @@ export class PAIJobInfoCollector {
}; };
//TODO : pass in request timeout param? //TODO : pass in request timeout param?
request(getJobInfoRequest, (error: Error, response: request.Response, body: any) => { request(getJobInfoRequest, (error: Error, response: request.Response, _body: any) => {
if ((error !== undefined && error !== null) || response.statusCode >= 500) { if ((error !== undefined && error !== null) || response.statusCode >= 500) {
this.log.error(`PAI Training service: get job info for trial ${paiTrialJob.id} from PAI Cluster failed!`); this.log.error(`PAI Training service: get job info for trial ${paiTrialJob.id} from PAI Cluster failed!`);
// Queried PAI job info failed, set job status to UNKNOWN // Queried PAI job info failed, set job status to UNKNOWN
......
...@@ -4,8 +4,6 @@ ...@@ -4,8 +4,6 @@
'use strict'; 'use strict';
import { Request, Response, Router } from 'express'; import { Request, Response, Router } from 'express';
import { Inject } from 'typescript-ioc';
import * as component from '../../common/component';
import { ClusterJobRestServer } from '../common/clusterJobRestServer'; import { ClusterJobRestServer } from '../common/clusterJobRestServer';
import { PAITrainingService } from './paiTrainingService'; import { PAITrainingService } from './paiTrainingService';
......
...@@ -19,7 +19,6 @@ ...@@ -19,7 +19,6 @@
'use strict'; 'use strict';
import * as cpp from 'child-process-promise';
import * as fs from 'fs'; import * as fs from 'fs';
import * as path from 'path'; import * as path from 'path';
// tslint:disable-next-line:no-implicit-dependencies // tslint:disable-next-line:no-implicit-dependencies
...@@ -29,11 +28,13 @@ import * as component from '../../../common/component'; ...@@ -29,11 +28,13 @@ import * as component from '../../../common/component';
import { Deferred } from 'ts-deferred'; import { Deferred } from 'ts-deferred';
import { String } from 'typescript-string-operations'; import { String } from 'typescript-string-operations';
import { import {
HyperParameters, NNIManagerIpConfig, TrainingService, HyperParameters, NNIManagerIpConfig,
TrialJobApplicationForm, TrialJobDetail, TrialJobMetric TrialJobApplicationForm, TrialJobDetail
} from '../../../common/trainingService'; } from '../../../common/trainingService';
import { delay, generateParamFileName, import {
getExperimentRootDir, getIPV4Address, getVersion, uniqueString, unixPathJoin } from '../../../common/utils'; generateParamFileName,
getIPV4Address, getVersion, uniqueString
} from '../../../common/utils';
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../../common/containerJobData'; import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../../common/containerJobData';
import { TrialConfigMetadataKey } from '../../common/trialConfigMetadataKey'; import { TrialConfigMetadataKey } from '../../common/trialConfigMetadataKey';
import { execMkdir, validateCodeDir, execCopydir } from '../../common/util'; import { execMkdir, validateCodeDir, execCopydir } from '../../common/util';
...@@ -56,7 +57,7 @@ class PAIK8STrainingService extends PAITrainingService { ...@@ -56,7 +57,7 @@ class PAIK8STrainingService extends PAITrainingService {
private nniVersion: string | undefined; private nniVersion: string | undefined;
constructor() { constructor() {
super(); super();
} }
public async setClusterMetadata(key: string, value: string): Promise<void> { public async setClusterMetadata(key: string, value: string): Promise<void> {
...@@ -69,10 +70,10 @@ class PAIK8STrainingService extends PAITrainingService { ...@@ -69,10 +70,10 @@ class PAIK8STrainingService extends PAITrainingService {
this.paiJobRestServer = new PAIJobRestServer(component.get(PAIK8STrainingService)); this.paiJobRestServer = new PAIJobRestServer(component.get(PAIK8STrainingService));
this.paiClusterConfig = <PAIClusterConfig>JSON.parse(value); this.paiClusterConfig = <PAIClusterConfig>JSON.parse(value);
this.paiClusterConfig.host = this.formatPAIHost(this.paiClusterConfig.host); this.paiClusterConfig.host = this.formatPAIHost(this.paiClusterConfig.host);
if(this.paiClusterConfig.passWord) { if (this.paiClusterConfig.passWord) {
// Get PAI authentication token // Get PAI authentication token
await this.updatePaiToken(); await this.updatePaiToken();
} else if(this.paiClusterConfig.token) { } else if (this.paiClusterConfig.token) {
this.paiToken = this.paiClusterConfig.token; this.paiToken = this.paiClusterConfig.token;
} }
break; break;
...@@ -104,7 +105,7 @@ class PAIK8STrainingService extends PAITrainingService { ...@@ -104,7 +105,7 @@ class PAIK8STrainingService extends PAITrainingService {
this.log.error(`Uknown key: ${key}`); this.log.error(`Uknown key: ${key}`);
} }
} }
// update trial parameters for multi-phase // update trial parameters for multi-phase
public async updateTrialJob(trialJobId: string, form: TrialJobApplicationForm): Promise<TrialJobDetail> { public async updateTrialJob(trialJobId: string, form: TrialJobApplicationForm): Promise<TrialJobDetail> {
const trialJobDetail: PAITrialJobDetail | undefined = this.trialJobsMap.get(trialJobId); const trialJobDetail: PAITrialJobDetail | undefined = this.trialJobsMap.get(trialJobId);
...@@ -146,8 +147,8 @@ class PAIK8STrainingService extends PAITrainingService { ...@@ -146,8 +147,8 @@ class PAIK8STrainingService extends PAITrainingService {
return trialJobDetail; return trialJobDetail;
} }
private generateNNITrialCommand(trialJobDetail: PAITrialJobDetail, command: string) { private generateNNITrialCommand(trialJobDetail: PAITrialJobDetail, command: string): string {
if (this.paiTrialConfig === undefined) { if (this.paiTrialConfig === undefined) {
throw new Error('trial config is not initialized'); throw new Error('trial config is not initialized');
} }
...@@ -167,13 +168,13 @@ class PAIK8STrainingService extends PAITrainingService { ...@@ -167,13 +168,13 @@ class PAIK8STrainingService extends PAITrainingService {
this.nniVersion, this.nniVersion,
this.logCollection this.logCollection
) )
.replace(/\r\n|\n|\r/gm, ''); .replace(/\r\n|\n|\r/gm, '');
return nniPaiTrialCommand; return nniPaiTrialCommand;
} }
private generateJobConfigInYamlFormat(trialJobDetail: PAITrialJobDetail) { private generateJobConfigInYamlFormat(trialJobDetail: PAITrialJobDetail): any {
if (this.paiTrialConfig === undefined) { if (this.paiTrialConfig === undefined) {
throw new Error('trial config is not initialized'); throw new Error('trial config is not initialized');
} }
...@@ -185,31 +186,31 @@ class PAIK8STrainingService extends PAITrainingService { ...@@ -185,31 +186,31 @@ class PAIK8STrainingService extends PAITrainingService {
nniJobConfig.name = jobName; nniJobConfig.name = jobName;
// Each taskRole will generate new command in NNI's command format // Each taskRole will generate new command in NNI's command format
// Each command will be formatted to NNI style // Each command will be formatted to NNI style
for(const taskRoleIndex in nniJobConfig.taskRoles) { for (const taskRoleIndex in nniJobConfig.taskRoles) {
const commands = nniJobConfig.taskRoles[taskRoleIndex].commands const commands = nniJobConfig.taskRoles[taskRoleIndex].commands
const nniTrialCommand = this.generateNNITrialCommand(trialJobDetail, commands.join(" && ").replace(/(["'$`\\])/g,'\\$1')); const nniTrialCommand = this.generateNNITrialCommand(trialJobDetail, commands.join(" && ").replace(/(["'$`\\])/g, '\\$1'));
nniJobConfig.taskRoles[taskRoleIndex].commands = [nniTrialCommand] nniJobConfig.taskRoles[taskRoleIndex].commands = [nniTrialCommand]
} }
} else { } else {
nniJobConfig = { nniJobConfig = {
protocolVersion: 2, protocolVersion: 2,
name: jobName, name: jobName,
type: 'job', type: 'job',
jobRetryCount: 0, jobRetryCount: 0,
prerequisites: [ prerequisites: [
{ {
type: 'dockerimage', type: 'dockerimage',
uri: this.paiTrialConfig.image, uri: this.paiTrialConfig.image,
name: 'docker_image_0' name: 'docker_image_0'
} }
], ],
taskRoles: { taskRoles: {
taskrole: { taskrole: {
instances: 1, instances: 1,
completion: { completion: {
minFailedInstances: 1, minFailedInstances: 1,
minSucceededInstances: -1 minSucceededInstances: -1
}, },
taskRetryCount: 0, taskRetryCount: 0,
dockerImage: 'docker_image_0', dockerImage: 'docker_image_0',
......
...@@ -3,27 +3,21 @@ ...@@ -3,27 +3,21 @@
'use strict'; 'use strict';
import * as fs from 'fs';
import * as path from 'path'; import * as path from 'path';
import * as request from 'request'; import * as request from 'request';
import * as component from '../../common/component'; import * as component from '../../common/component';
import { EventEmitter } from 'events'; import { EventEmitter } from 'events';
import { Deferred } from 'ts-deferred'; import { Deferred } from 'ts-deferred';
import { String } from 'typescript-string-operations';
import { getExperimentId } from '../../common/experimentStartupInfo'; import { getExperimentId } from '../../common/experimentStartupInfo';
import { getLogger, Logger } from '../../common/log'; import { getLogger, Logger } from '../../common/log';
import { import {
HyperParameters, NNIManagerIpConfig, TrainingService, NNIManagerIpConfig, TrainingService,
TrialJobApplicationForm, TrialJobDetail, TrialJobMetric TrialJobApplicationForm, TrialJobDetail, TrialJobMetric
} from '../../common/trainingService'; } from '../../common/trainingService';
import { delay, generateParamFileName, import { delay } from '../../common/utils';
getExperimentRootDir, getIPV4Address, getVersion, uniqueString, unixPathJoin } from '../../common/utils';
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData';
import { TrialConfigMetadataKey } from '../common/trialConfigMetadataKey';
import { execMkdir, validateCodeDir } from '../common/util';
import { PAIJobInfoCollector } from './paiJobInfoCollector'; import { PAIJobInfoCollector } from './paiJobInfoCollector';
import { PAIJobRestServer, ParameterFileMeta } from './paiJobRestServer'; import { PAIJobRestServer } from './paiJobRestServer';
import { PAIClusterConfig, PAITrialJobDetail } from './paiConfig'; import { PAIClusterConfig, PAITrialJobDetail } from './paiConfig';
/** /**
...@@ -39,7 +33,7 @@ abstract class PAITrainingService implements TrainingService { ...@@ -39,7 +33,7 @@ abstract class PAITrainingService implements TrainingService {
protected paiClusterConfig?: PAIClusterConfig; protected paiClusterConfig?: PAIClusterConfig;
protected readonly jobQueue: string[]; protected readonly jobQueue: string[];
protected stopping: boolean = false; protected stopping: boolean = false;
protected paiToken? : string; protected paiToken?: string;
protected paiTokenUpdateTime?: number; protected paiTokenUpdateTime?: number;
protected readonly paiTokenUpdateInterval: number; protected readonly paiTokenUpdateInterval: number;
protected readonly experimentId!: string; protected readonly experimentId!: string;
...@@ -81,15 +75,15 @@ abstract class PAITrainingService implements TrainingService { ...@@ -81,15 +75,15 @@ abstract class PAITrainingService implements TrainingService {
this.log.info('PAI training service exit.'); this.log.info('PAI training service exit.');
} }
public async submitTrialJob(form: TrialJobApplicationForm): Promise<any> { public async submitTrialJob(_form: TrialJobApplicationForm): Promise<any> {
throw new Error('Not implemented!'); throw new Error('Not implemented!');
} }
public async updateTrialJob(trialJobId: string, form: TrialJobApplicationForm): Promise<TrialJobDetail> { public async updateTrialJob(_trialJobId: string, _form: TrialJobApplicationForm): Promise<TrialJobDetail> {
throw new Error('Not implemented!'); throw new Error('Not implemented!');
} }
protected async submitTrialJobToPAI(trialJobId: string): Promise<boolean> { protected async submitTrialJobToPAI(_trialJobId: string): Promise<boolean> {
throw new Error('Not implemented!'); throw new Error('Not implemented!');
} }
...@@ -109,14 +103,14 @@ abstract class PAITrainingService implements TrainingService { ...@@ -109,14 +103,14 @@ abstract class PAITrainingService implements TrainingService {
} }
} }
public async setClusterMetadata(key: string, value: string): Promise<void> { public async setClusterMetadata(_key: string, _value: string): Promise<void> {
throw new Error('Not implemented!'); throw new Error('Not implemented!');
} }
public async listTrialJobs(): Promise<TrialJobDetail[]> { public async listTrialJobs(): Promise<TrialJobDetail[]> {
const jobs: TrialJobDetail[] = []; const jobs: TrialJobDetail[] = [];
for (const [key, value] of this.trialJobsMap) { for (const key of this.trialJobsMap.keys()) {
jobs.push(await this.getTrialJob(key)); jobs.push(await this.getTrialJob(key));
} }
...@@ -150,7 +144,7 @@ abstract class PAITrainingService implements TrainingService { ...@@ -150,7 +144,7 @@ abstract class PAITrainingService implements TrainingService {
} }
public cancelTrialJob(trialJobId: string, isEarlyStopped: boolean = false): Promise<void> { public cancelTrialJob(trialJobId: string, isEarlyStopped: boolean = false): Promise<void> {
const trialJobDetail: PAITrialJobDetail | undefined = this.trialJobsMap.get(trialJobId); const trialJobDetail: PAITrialJobDetail | undefined = this.trialJobsMap.get(trialJobId);
if (trialJobDetail === undefined) { if (trialJobDetail === undefined) {
return Promise.reject(new Error(`cancelTrialJob: trial job id ${trialJobId} not found`)); return Promise.reject(new Error(`cancelTrialJob: trial job id ${trialJobId} not found`));
} }
...@@ -169,10 +163,10 @@ abstract class PAITrainingService implements TrainingService { ...@@ -169,10 +163,10 @@ abstract class PAITrainingService implements TrainingService {
const stopJobRequest: request.Options = { const stopJobRequest: request.Options = {
uri: `${this.protocol}://${this.paiClusterConfig.host}/rest-server/api/v1/user/${this.paiClusterConfig.userName}\ uri: `${this.protocol}://${this.paiClusterConfig.host}/rest-server/api/v1/user/${this.paiClusterConfig.userName}\
/jobs/${trialJobDetail.paiJobName}/executionType`, /jobs/${trialJobDetail.paiJobName}/executionType`,
method: 'PUT', method: 'PUT',
json: true, json: true,
body: {value: 'STOP'}, body: { value: 'STOP' },
headers: { headers: {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
Authorization: `Bearer ${this.paiToken}` Authorization: `Bearer ${this.paiToken}`
...@@ -183,11 +177,11 @@ abstract class PAITrainingService implements TrainingService { ...@@ -183,11 +177,11 @@ abstract class PAITrainingService implements TrainingService {
trialJobDetail.isEarlyStopped = isEarlyStopped; trialJobDetail.isEarlyStopped = isEarlyStopped;
const deferred: Deferred<void> = new Deferred<void>(); const deferred: Deferred<void> = new Deferred<void>();
request(stopJobRequest, (error: Error, response: request.Response, body: any) => { request(stopJobRequest, (error: Error, response: request.Response, _body: any) => {
if ((error !== undefined && error !== null) || response.statusCode >= 400) { if ((error !== undefined && error !== null) || response.statusCode >= 400) {
this.log.error(`PAI Training service: stop trial ${trialJobId} to PAI Cluster failed!`); this.log.error(`PAI Training service: stop trial ${trialJobId} to PAI Cluster failed!`);
deferred.reject((error !== undefined && error !== null) ? error.message : deferred.reject((error !== undefined && error !== null) ? error.message :
`Stop trial failed, http code: ${response.statusCode}`); `Stop trial failed, http code: ${response.statusCode}`);
} else { } else {
deferred.resolve(); deferred.resolve();
} }
...@@ -196,7 +190,7 @@ abstract class PAITrainingService implements TrainingService { ...@@ -196,7 +190,7 @@ abstract class PAITrainingService implements TrainingService {
return deferred.promise; return deferred.promise;
} }
public getClusterMetadata(key: string): Promise<string> { public getClusterMetadata(_key: string): Promise<string> {
throw new Error('Not implemented!'); throw new Error('Not implemented!');
} }
...@@ -236,7 +230,7 @@ abstract class PAITrainingService implements TrainingService { ...@@ -236,7 +230,7 @@ abstract class PAITrainingService implements TrainingService {
protected async statusCheckingLoop(): Promise<void> { protected async statusCheckingLoop(): Promise<void> {
while (!this.stopping) { while (!this.stopping) {
if(this.paiClusterConfig && this.paiClusterConfig.passWord) { if (this.paiClusterConfig && this.paiClusterConfig.passWord) {
try { try {
await this.updatePaiToken(); await this.updatePaiToken();
} catch (error) { } catch (error) {
...@@ -302,7 +296,7 @@ abstract class PAITrainingService implements TrainingService { ...@@ -302,7 +296,7 @@ abstract class PAITrainingService implements TrainingService {
}); });
let timeoutId: NodeJS.Timer; let timeoutId: NodeJS.Timer;
const timeoutDelay: Promise<void> = new Promise<void>((resolve: Function, reject: Function): void => { const timeoutDelay: Promise<void> = new Promise<void>((_resolve: Function, reject: Function): void => {
// Set timeout and reject the promise once reach timeout (5 seconds) // Set timeout and reject the promise once reach timeout (5 seconds)
timeoutId = setTimeout( timeoutId = setTimeout(
() => reject(new Error('Get PAI token timeout. Please check your PAI cluster.')), () => reject(new Error('Get PAI token timeout. Please check your PAI cluster.')),
......
...@@ -3,8 +3,6 @@ ...@@ -3,8 +3,6 @@
'use strict'; 'use strict';
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../../common/trainingService';
export const PAI_INSTALL_NNI_SHELL_FORMAT: string = export const PAI_INSTALL_NNI_SHELL_FORMAT: string =
`#!/bin/bash `#!/bin/bash
if python3 -c 'import nni' > /dev/null 2>&1; then if python3 -c 'import nni' > /dev/null 2>&1; then
......
...@@ -8,24 +8,22 @@ import * as path from 'path'; ...@@ -8,24 +8,22 @@ import * as path from 'path';
import * as request from 'request'; import * as request from 'request';
import * as component from '../../../common/component'; import * as component from '../../../common/component';
import { EventEmitter } from 'events';
import { Deferred } from 'ts-deferred'; import { Deferred } from 'ts-deferred';
import { String } from 'typescript-string-operations'; import { String } from 'typescript-string-operations';
import { getExperimentId } from '../../../common/experimentStartupInfo';
import { getLogger, Logger } from '../../../common/log';
import { import {
HyperParameters, NNIManagerIpConfig, TrainingService, HyperParameters, NNIManagerIpConfig,
TrialJobApplicationForm, TrialJobDetail, TrialJobMetric TrialJobApplicationForm, TrialJobDetail
} from '../../../common/trainingService'; } from '../../../common/trainingService';
import { delay, generateParamFileName, import {
getExperimentRootDir, getIPV4Address, getVersion, uniqueString, unixPathJoin } from '../../../common/utils'; generateParamFileName,
getExperimentRootDir, getIPV4Address, getVersion, uniqueString, unixPathJoin
} from '../../../common/utils';
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../../common/containerJobData'; import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../../common/containerJobData';
import { TrialConfigMetadataKey } from '../../common/trialConfigMetadataKey'; import { TrialConfigMetadataKey } from '../../common/trialConfigMetadataKey';
import { execMkdir, validateCodeDir } from '../../common/util'; import { execMkdir, validateCodeDir } from '../../common/util';
import { HDFSClientUtility } from './hdfsClientUtility'; import { HDFSClientUtility } from './hdfsClientUtility';
import { NNIPAITrialConfig, PAIJobConfig, PAITaskRole } from './paiYarnConfig'; import { NNIPAITrialConfig, PAIJobConfig, PAITaskRole } from './paiYarnConfig';
import { PAI_LOG_PATH_FORMAT, PAI_TRIAL_COMMAND_FORMAT } from './paiYarnData'; import { PAI_LOG_PATH_FORMAT, PAI_TRIAL_COMMAND_FORMAT } from './paiYarnData';
import { PAIJobInfoCollector } from '../paiJobInfoCollector';
import { PAITrainingService } from '../paiTrainingService'; import { PAITrainingService } from '../paiTrainingService';
import { PAIClusterConfig, PAITrialJobDetail } from '../paiConfig'; import { PAIClusterConfig, PAITrialJobDetail } from '../paiConfig';
...@@ -65,7 +63,7 @@ class PAIYarnTrainingService extends PAITrainingService { ...@@ -65,7 +63,7 @@ class PAIYarnTrainingService extends PAITrainingService {
PAI_LOG_PATH_FORMAT, PAI_LOG_PATH_FORMAT,
this.paiClusterConfig.host, this.paiClusterConfig.host,
hdfsOutputDir hdfsOutputDir
); );
const trialJobDetail: PAITrialJobDetail = new PAITrialJobDetail( const trialJobDetail: PAITrialJobDetail = new PAITrialJobDetail(
trialJobId, trialJobId,
...@@ -99,13 +97,13 @@ class PAIYarnTrainingService extends PAITrainingService { ...@@ -99,13 +97,13 @@ class PAIYarnTrainingService extends PAITrainingService {
port: 80, port: 80,
path: '/webhdfs/api/v1', path: '/webhdfs/api/v1',
host: this.paiClusterConfig.host host: this.paiClusterConfig.host
}); });
this.paiClusterConfig.host = this.formatPAIHost(this.paiClusterConfig.host); this.paiClusterConfig.host = this.formatPAIHost(this.paiClusterConfig.host);
if(this.paiClusterConfig.passWord) { if (this.paiClusterConfig.passWord) {
// Get PAI authentication token // Get PAI authentication token
await this.updatePaiToken(); await this.updatePaiToken();
} else if(this.paiClusterConfig.token) { } else if (this.paiClusterConfig.token) {
this.paiToken = this.paiClusterConfig.token; this.paiToken = this.paiClusterConfig.token;
} else { } else {
throw new Error('pai cluster config format error, please set password or token!'); throw new Error('pai cluster config format error, please set password or token!');
...@@ -121,14 +119,14 @@ class PAIYarnTrainingService extends PAITrainingService { ...@@ -121,14 +119,14 @@ class PAIYarnTrainingService extends PAITrainingService {
// Validate to make sure codeDir doesn't have too many files // Validate to make sure codeDir doesn't have too many files
await validateCodeDir(this.paiTrialConfig.codeDir); await validateCodeDir(this.paiTrialConfig.codeDir);
// Copy experiment files from local folder to HDFS // Copy experiment files from local folder to HDFS
this.copyExpCodeDirPromise = HDFSClientUtility.copyDirectoryToHdfs( this.copyExpCodeDirPromise = HDFSClientUtility.copyDirectoryToHdfs(
this.paiTrialConfig.codeDir, this.paiTrialConfig.codeDir,
HDFSClientUtility.getHdfsExpCodeDir(this.paiClusterConfig.userName), HDFSClientUtility.getHdfsExpCodeDir(this.paiClusterConfig.userName),
this.hdfsClient this.hdfsClient
); );
// Upload authFile to hdfs // Upload authFile to hdfs
if (this.paiTrialConfig.authFile) { if (this.paiTrialConfig.authFile) {
this.authFileHdfsPath = unixPathJoin(HDFSClientUtility.hdfsExpRootDir(this.paiClusterConfig.userName), 'authFile'); this.authFileHdfsPath = unixPathJoin(HDFSClientUtility.hdfsExpRootDir(this.paiClusterConfig.userName), 'authFile');
...@@ -224,7 +222,7 @@ class PAIYarnTrainingService extends PAITrainingService { ...@@ -224,7 +222,7 @@ class PAIYarnTrainingService extends PAITrainingService {
version, version,
this.logCollection this.logCollection
) )
.replace(/\r\n|\n|\r/gm, ''); .replace(/\r\n|\n|\r/gm, '');
this.log.info(`nniPAItrial command is ${nniPaiTrialCommand.trim()}`); this.log.info(`nniPAItrial command is ${nniPaiTrialCommand.trim()}`);
const paiTaskRoles: PAITaskRole[] = [ const paiTaskRoles: PAITaskRole[] = [
...@@ -283,10 +281,8 @@ class PAIYarnTrainingService extends PAITrainingService { ...@@ -283,10 +281,8 @@ class PAIYarnTrainingService extends PAITrainingService {
Authorization: `Bearer ${this.paiToken}` Authorization: `Bearer ${this.paiToken}`
} }
}; };
request(submitJobRequest, (error: Error, response: request.Response, body: any) => { request(submitJobRequest, (error: Error, response: request.Response, _body: any) => {
if ((error !== undefined && error !== null) || response.statusCode >= 400) { if ((error !== undefined && error !== null) || response.statusCode >= 400) {
const errorMessage: string = (error !== undefined && error !== null) ? error.message :
`Submit trial ${trialJobId} failed, http code:${response.statusCode}, http body: ${response.body.message}`;
trialJobDetail.status = 'FAILED'; trialJobDetail.status = 'FAILED';
deferred.resolve(true); deferred.resolve(true);
} else { } else {
...@@ -343,7 +339,7 @@ class PAIYarnTrainingService extends PAITrainingService { ...@@ -343,7 +339,7 @@ class PAIYarnTrainingService extends PAITrainingService {
json: true, json: true,
body: parameterFileMeta body: parameterFileMeta
}; };
request(req, (err: Error, res: request.Response) => { request(req, (err: Error, _res: request.Response) => {
if (err) { if (err) {
deferred.reject(err); deferred.reject(err);
} else { } else {
......
...@@ -351,7 +351,7 @@ class RemoteMachineTrainingService implements TrainingService { ...@@ -351,7 +351,7 @@ class RemoteMachineTrainingService implements TrainingService {
* Get culster metadata * Get culster metadata
* @param key metadata key * @param key metadata key
*/ */
public async getClusterMetadata(key: string): Promise<string> { public async getClusterMetadata(_key: string): Promise<string> {
return ""; return "";
} }
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment