Skip to content

GitLab

Unverified Commit 817ec68b authored by liuzhe-lz's avatar liuzhe-lz Committed by GitHub
Browse files

Add native support for v2 config (#3466)

parent 6aaca5f7
Expand all Show whitespace changes
Inline Side-by-side
Showing with 382 additions and 773 deletions
ts/nni_manager/training_service/remote_machine/remoteMachineData.ts
View file @ 817ec68b
......@@ -4,6 +4,7 @@
'use strict';
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService';
import { RemoteMachineConfig } from '../../common/experimentConfig';
import { GPUInfo, GPUSummary, ScheduleResultType } from '../common/gpuData';
import { ShellExecutor } from './shellExecutor';
......@@ -11,19 +12,14 @@ import { ShellExecutor } from './shellExecutor';
* Metadata of remote machine for configuration and statuc query
*/
export class RemoteMachineMeta {
public readonly ip: string = '';
public readonly port: number = 22;
public readonly username: string = '';
public readonly passwd: string = '';
public readonly sshKeyPath?: string;
public readonly passphrase?: string;
public readonly config: RemoteMachineConfig;
public gpuSummary: GPUSummary | undefined;
public readonly gpuIndices?: string;
public readonly maxTrialNumPerGpu?: number;
//TODO: initialize varialbe in constructor
public occupiedGpuIndexMap?: Map<number, number>;
public readonly useActiveGpu?: boolean = false;
public readonly pythonPath?: string;
public occupiedGpuIndexMap: Map<number, number>;
constructor(config: RemoteMachineConfig) {
this.config = config;
this.occupiedGpuIndexMap = new Map<number, number>();
}
}
/**
......@@ -74,13 +70,13 @@ export class RemoteMachineTrialJobDetail implements TrialJobDetail {
* The remote machine executor manager
*/
export class ExecutorManager {
public readonly rmMeta: RemoteMachineMeta;
private readonly executorMap: Map<string, ShellExecutor> = new Map<string, ShellExecutor>();
private readonly rmMeta: RemoteMachineMeta;
private executors: ShellExecutor[] = [];
constructor(rmMeta: RemoteMachineMeta) {
this.rmMeta = rmMeta;
constructor(config: RemoteMachineConfig) {
this.rmMeta = new RemoteMachineMeta(config);
}
public async getExecutor(id: string): Promise<ShellExecutor> {
......
ts/nni_manager/training_service/remote_machine/remoteMachineJobRestServer.ts
View file @ 817ec68b
......@@ -3,8 +3,6 @@
'use strict';
import { Inject } from 'typescript-ioc';
import * as component from '../../common/component';
import { ClusterJobRestServer } from '../common/clusterJobRestServer';
import { RemoteMachineTrainingService } from './remoteMachineTrainingService';
......@@ -12,17 +10,15 @@ import { RemoteMachineTrainingService } from './remoteMachineTrainingService';
* RemoteMachine Training service Rest server, provides rest RemoteMachine to support remotemachine job metrics update
*
*/
@component.Singleton
export class RemoteMachineJobRestServer extends ClusterJobRestServer {
@Inject
private readonly remoteMachineTrainingService: RemoteMachineTrainingService;
/**
* constructor to provide NNIRestServer's own rest property, e.g. port
*/
constructor() {
constructor(remoteMachineTrainingService: RemoteMachineTrainingService) {
super();
this.remoteMachineTrainingService = component.get(RemoteMachineTrainingService);
this.remoteMachineTrainingService = remoteMachineTrainingService;
}
protected handleTrialMetrics(jobId: string, metrics: any[]): void {
......
ts/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts
View file @ 817ec68b
......@@ -15,70 +15,77 @@ import { getExperimentId } from '../../common/experimentStartupInfo';
import { getLogger, Logger } from '../../common/log';
import { ObservableTimer } from '../../common/observableTimer';
import {
HyperParameters, NNIManagerIpConfig, TrainingService, TrialJobApplicationForm,
HyperParameters, TrainingService, TrialJobApplicationForm,
TrialJobDetail, TrialJobMetric, LogType
} from '../../common/trainingService';
import {
delay, generateParamFileName, getExperimentRootDir, getIPV4Address, getJobCancelStatus,
getVersion, uniqueString
} from '../../common/utils';
import { ExperimentConfig, RemoteConfig, RemoteMachineConfig, flattenConfig } from '../../common/experimentConfig';
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData';
import { GPUSummary, ScheduleResultType } from '../common/gpuData';
import { TrialConfig } from '../common/trialConfig';
import { TrialConfigMetadataKey } from '../common/trialConfigMetadataKey';
import { execMkdir, validateCodeDir } from '../common/util';
import { GPUScheduler } from './gpuScheduler';
import {
ExecutorManager, RemoteMachineMeta,
RemoteMachineScheduleInfo, RemoteMachineScheduleResult, RemoteMachineTrialJobDetail
ExecutorManager, RemoteMachineScheduleInfo, RemoteMachineScheduleResult, RemoteMachineTrialJobDetail
} from './remoteMachineData';
import { RemoteMachineJobRestServer } from './remoteMachineJobRestServer';
interface FlattenRemoteConfig extends ExperimentConfig, RemoteConfig { }
/**
* Training Service implementation for Remote Machine (Linux)
*/
@component.Singleton
class RemoteMachineTrainingService implements TrainingService {
private readonly initExecutorId = "initConnection";
private readonly machineExecutorManagerMap: Map<RemoteMachineMeta, ExecutorManager>; //machine excutor map
private readonly machineCopyExpCodeDirPromiseMap: Map<RemoteMachineMeta, Promise<void>>;
private readonly machineExecutorManagerMap: Map<RemoteMachineConfig, ExecutorManager>; //machine excutor map
private readonly machineCopyExpCodeDirPromiseMap: Map<RemoteMachineConfig, Promise<void>>;
private readonly trialExecutorManagerMap: Map<string, ExecutorManager>; //trial excutor map
private readonly trialJobsMap: Map<string, RemoteMachineTrialJobDetail>;
private readonly expRootDir: string;
private trialConfig: TrialConfig | undefined;
private gpuScheduler?: GPUScheduler;
private readonly jobQueue: string[];
private readonly timer: ObservableTimer;
private stopping: boolean = false;
private readonly metricsEmitter: EventEmitter;
private readonly log: Logger;
private isMultiPhase: boolean = false;
private remoteRestServerPort?: number;
private nniManagerIpConfig?: NNIManagerIpConfig;
private versionCheck: boolean = true;
private logCollection: string;
private logCollection: string = 'none';
private sshConnectionPromises: any[];
private config: FlattenRemoteConfig;
constructor(@component.Inject timer: ObservableTimer) {
constructor(config: ExperimentConfig) {
this.metricsEmitter = new EventEmitter();
this.trialJobsMap = new Map<string, RemoteMachineTrialJobDetail>();
this.trialExecutorManagerMap = new Map<string, ExecutorManager>();
this.machineCopyExpCodeDirPromiseMap = new Map<RemoteMachineMeta, Promise<void>>();
this.machineExecutorManagerMap = new Map<RemoteMachineMeta, ExecutorManager>();
this.machineCopyExpCodeDirPromiseMap = new Map<RemoteMachineConfig, Promise<void>>();
this.machineExecutorManagerMap = new Map<RemoteMachineConfig, ExecutorManager>();
this.jobQueue = [];
this.sshConnectionPromises = [];
this.expRootDir = getExperimentRootDir();
this.timer = timer;
this.timer = component.get(ObservableTimer);
this.log = getLogger();
this.logCollection = 'none';
this.log.info('Construct remote machine training service.');
this.config = flattenConfig(config, 'remote');
if (!fs.lstatSync(this.config.trialCodeDirectory).isDirectory()) {
throw new Error(`codeDir ${this.config.trialCodeDirectory} is not a directory`);
}
validateCodeDir(this.config.trialCodeDirectory);
this.sshConnectionPromises = this.config.machineList.map(
machine => this.initRemoteMachineOnConnected(machine)
);
}
/**
* Loop to launch trial jobs and collect trial metrics
*/
public async run(): Promise<void> {
const restServer: RemoteMachineJobRestServer = component.get(RemoteMachineJobRestServer);
const restServer = new RemoteMachineJobRestServer(this);
await restServer.start();
restServer.setEnableVersionCheck = this.versionCheck;
this.log.info('Run remote machine training service.');
......@@ -89,16 +96,13 @@ class RemoteMachineTrainingService implements TrainingService {
this.sshConnectionPromises = [];
// initialize gpuScheduler
this.gpuScheduler = new GPUScheduler(this.machineExecutorManagerMap);
if (this.trialConfig === undefined) {
throw new Error("trial config not initialized!");
}
// Copy codeDir to remote machine
for (const [rmMeta, executorManager] of this.machineExecutorManagerMap.entries()) {
for (const [machineConfig, executorManager] of this.machineExecutorManagerMap.entries()) {
const executor: ShellExecutor = await executorManager.getExecutor(this.initExecutorId);
if (executor !== undefined) {
this.machineCopyExpCodeDirPromiseMap.set(
rmMeta,
executor.copyDirectoryToRemote(this.trialConfig.codeDir, executor.getRemoteCodePath(getExperimentId()))
machineConfig,
executor.copyDirectoryToRemote(this.config.trialCodeDirectory, executor.getRemoteCodePath(getExperimentId()))
);
}
}
......@@ -134,7 +138,7 @@ class RemoteMachineTrainingService implements TrainingService {
if (trial.rmMeta === undefined) {
throw new Error(`rmMeta not set in trial ${trial.id}`);
}
const executorManager: ExecutorManager | undefined = this.machineExecutorManagerMap.get(trial.rmMeta);
const executorManager: ExecutorManager | undefined = this.machineExecutorManagerMap.get(trial.rmMeta.config);
if (executorManager === undefined) {
throw new Error(`executorManager not initialized`);
}
......@@ -225,10 +229,6 @@ class RemoteMachineTrainingService implements TrainingService {
* @param form trial job description form
*/
public async submitTrialJob(form: TrialJobApplicationForm): Promise<TrialJobDetail> {
if (this.trialConfig === undefined) {
throw new Error('trial config is not initialized');
}
// Generate trial job id(random)
const trialJobId: string = uniqueString(5);
......@@ -260,13 +260,6 @@ class RemoteMachineTrainingService implements TrainingService {
return trialJobDetail;
}
/**
* Is multiphase job supported in current training service
*/
public get isMultiPhaseJobSupported(): boolean {
return true;
}
/**
* Cancel trial job
* @param trialJobId ID of trial job
......@@ -311,70 +304,8 @@ class RemoteMachineTrainingService implements TrainingService {
}
}
/**
* Set culster metadata
* @param key metadata key
* //1. MACHINE_LIST -- create executor of machine list
* //2. TRIAL_CONFIG -- trial configuration
* @param value metadata value
*/
public async setClusterMetadata(key: string, value: string): Promise<void> {
switch (key) {
case TrialConfigMetadataKey.NNI_MANAGER_IP:
this.nniManagerIpConfig = <NNIManagerIpConfig>JSON.parse(value);
break;
case TrialConfigMetadataKey.MACHINE_LIST:
await this.setupConnections(value);
break;
case TrialConfigMetadataKey.TRIAL_CONFIG: {
const remoteMachineTrailConfig: TrialConfig = <TrialConfig>JSON.parse(value);
// Parse trial config failed, throw Error
if (remoteMachineTrailConfig === undefined) {
throw new Error('trial config parsed failed');
}
// codeDir is not a valid directory, throw Error
if (!fs.lstatSync(remoteMachineTrailConfig.codeDir)
.isDirectory()) {
throw new Error(`codeDir ${remoteMachineTrailConfig.codeDir} is not a directory`);
}
try {
// Validate to make sure codeDir doesn't have too many files
await validateCodeDir(remoteMachineTrailConfig.codeDir);
} catch (error) {
this.log.error(error);
return Promise.reject(new Error(error));
}
this.trialConfig = remoteMachineTrailConfig;
break;
}
case TrialConfigMetadataKey.MULTI_PHASE:
this.isMultiPhase = (value === 'true' || value === 'True');
break;
case TrialConfigMetadataKey.VERSION_CHECK:
this.versionCheck = (value === 'true' || value === 'True');
break;
case TrialConfigMetadataKey.LOG_COLLECTION:
this.logCollection = value;
break;
case TrialConfigMetadataKey.REMOTE_CONFIG:
// Add remote_config in remoteEnvironmentService to set reuse mode,
// this config need to be catched here, otherwise will throw Unknown key exception here
break;
default:
//Reject for unknown keys
throw new Error(`Uknown key: ${key}`);
}
}
/**
* Get culster metadata
* @param key metadata key
*/
public async getClusterMetadata(_key: string): Promise<string> {
return "";
}
public async setClusterMetadata(_key: string, _value: string): Promise<void> { return; }
public async getClusterMetadata(_key: string): Promise<string> { return ''; }
/**
* cleanup() has a time out of 10s to clean remote connections
......@@ -426,23 +357,12 @@ class RemoteMachineTrainingService implements TrainingService {
}
}
private async setupConnections(machineList: string): Promise<void> {
this.log.debug(`Connecting to remote machines: ${machineList}`);
//TO DO: verify if value's format is wrong, and json parse failed, how to handle error
const rmMetaList: RemoteMachineMeta[] = <RemoteMachineMeta[]>JSON.parse(machineList);
for (const rmMeta of rmMetaList) {
this.sshConnectionPromises.push(this.initRemoteMachineOnConnected(rmMeta));
}
}
private async initRemoteMachineOnConnected(rmMeta: RemoteMachineMeta): Promise<void> {
rmMeta.occupiedGpuIndexMap = new Map<number, number>();
const executorManager: ExecutorManager = new ExecutorManager(rmMeta);
this.log.info(`connecting to ${rmMeta.username}@${rmMeta.ip}:${rmMeta.port}`);
private async initRemoteMachineOnConnected(machineConfig: RemoteMachineConfig): Promise<void> {
const executorManager: ExecutorManager = new ExecutorManager(machineConfig);
this.log.info(`connecting to ${machineConfig.user}@${machineConfig.host}:${machineConfig.port}`);
const executor: ShellExecutor = await executorManager.getExecutor(this.initExecutorId);
this.log.debug(`reached ${executor.name}`);
this.machineExecutorManagerMap.set(rmMeta, executorManager);
this.machineExecutorManagerMap.set(machineConfig, executorManager);
this.log.debug(`initializing ${executor.name}`);
// Create root working directory after executor is ready
......@@ -469,15 +389,15 @@ class RemoteMachineTrainingService implements TrainingService {
collectingCount.push(true);
const cmdresult = await executor.readLastLines(executor.joinPath(remoteGpuScriptCollectorDir, 'gpu_metrics'));
if (cmdresult !== "") {
rmMeta.gpuSummary = <GPUSummary>JSON.parse(cmdresult);
if (rmMeta.gpuSummary.gpuCount === 0) {
this.log.warning(`No GPU found on remote machine ${rmMeta.ip}`);
executorManager.rmMeta.gpuSummary = <GPUSummary>JSON.parse(cmdresult);
if (executorManager.rmMeta.gpuSummary.gpuCount === 0) {
this.log.warning(`No GPU found on remote machine ${machineConfig.host}`);
this.timer.unsubscribe(disposable);
}
}
if (this.stopping) {
this.timer.unsubscribe(disposable);
this.log.debug(`Stopped GPU collector on ${rmMeta.ip}, since experiment is exiting.`);
this.log.debug(`Stopped GPU collector on ${machineConfig.host}, since experiment is exiting.`);
}
collectingCount.pop();
}
......@@ -488,9 +408,6 @@ class RemoteMachineTrainingService implements TrainingService {
private async prepareTrialJob(trialJobId: string): Promise<boolean> {
const deferred: Deferred<boolean> = new Deferred<boolean>();
if (this.trialConfig === undefined) {
throw new Error('trial config is not initialized');
}
if (this.gpuScheduler === undefined) {
throw new Error('gpuScheduler is not initialized');
}
......@@ -505,9 +422,9 @@ class RemoteMachineTrainingService implements TrainingService {
return deferred.promise;
}
// get an executor from scheduler
const rmScheduleResult: RemoteMachineScheduleResult = this.gpuScheduler.scheduleMachine(this.trialConfig.gpuNum, trialJobDetail);
const rmScheduleResult: RemoteMachineScheduleResult = this.gpuScheduler.scheduleMachine(this.config.trialGpuNumber, trialJobDetail);
if (rmScheduleResult.resultType === ScheduleResultType.REQUIRE_EXCEED_TOTAL) {
const errorMessage: string = `Required GPU number ${this.trialConfig.gpuNum} is too large, no machine can meet`;
const errorMessage: string = `Required GPU number ${this.config.trialGpuNumber} is too large, no machine can meet`;
this.log.error(errorMessage);
deferred.reject();
throw new NNIError(NNIErrorNames.RESOURCE_NOT_AVAILABLE, errorMessage);
......@@ -516,7 +433,7 @@ class RemoteMachineTrainingService implements TrainingService {
const rmScheduleInfo: RemoteMachineScheduleInfo = rmScheduleResult.scheduleInfo;
trialJobDetail.rmMeta = rmScheduleInfo.rmMeta;
const copyExpCodeDirPromise = this.machineCopyExpCodeDirPromiseMap.get(trialJobDetail.rmMeta);
const copyExpCodeDirPromise = this.machineCopyExpCodeDirPromiseMap.get(rmScheduleInfo.rmMeta.config);
if (copyExpCodeDirPromise !== undefined) {
await copyExpCodeDirPromise;
}
......@@ -530,7 +447,7 @@ class RemoteMachineTrainingService implements TrainingService {
trialJobId, trialJobDetail.form, rmScheduleInfo);
trialJobDetail.status = 'RUNNING';
trialJobDetail.url = `file://${rmScheduleInfo.rmMeta.ip}:${trialJobDetail.workingDirectory}`;
trialJobDetail.url = `file://${rmScheduleInfo.rmMeta.config.host}:${trialJobDetail.workingDirectory}`;
trialJobDetail.startTime = Date.now();
this.trialJobsMap.set(trialJobId, trialJobDetail);
......@@ -547,9 +464,6 @@ class RemoteMachineTrainingService implements TrainingService {
private async launchTrialOnScheduledMachine(trialJobId: string, form: TrialJobApplicationForm,
rmScheduleInfo: RemoteMachineScheduleInfo): Promise<void> {
if (this.trialConfig === undefined) {
throw new Error('trial config is not initialized');
}
const cudaVisibleDevice: string = rmScheduleInfo.cudaVisibleDevice;
const executor = await this.getExecutor(trialJobId);
const trialJobDetail: RemoteMachineTrialJobDetail | undefined = this.trialJobsMap.get(trialJobId);
......@@ -568,7 +482,7 @@ class RemoteMachineTrainingService implements TrainingService {
// Set CUDA_VISIBLE_DEVICES environment variable based on cudaVisibleDevice
// If no valid cudaVisibleDevice is defined, set CUDA_VISIBLE_DEVICES to empty string to hide GPU device
// If gpuNum is undefined, will not set CUDA_VISIBLE_DEVICES in script
if (this.trialConfig.gpuNum === undefined) {
if (this.config.trialGpuNumber === undefined) {
cudaVisible = ""
} else {
if (typeof cudaVisibleDevice === 'string' && cudaVisibleDevice.length > 0) {
......@@ -577,7 +491,7 @@ class RemoteMachineTrainingService implements TrainingService {
cudaVisible = `CUDA_VISIBLE_DEVICES=" "`;
}
}
const nniManagerIp: string = this.nniManagerIpConfig ? this.nniManagerIpConfig.nniManagerIp : getIPV4Address();
const nniManagerIp: string = this.config.nniManagerIp ? this.config.nniManagerIp : getIPV4Address();
if (this.remoteRestServerPort === undefined) {
const restServer: RemoteMachineJobRestServer = component.get(RemoteMachineJobRestServer);
this.remoteRestServerPort = restServer.clusterRestServerPort;
......@@ -588,12 +502,13 @@ class RemoteMachineTrainingService implements TrainingService {
trialJobId,
getExperimentId(),
trialJobDetail.form.sequenceId.toString(),
this.isMultiPhase,
this.trialConfig.command,
false, // multi-phase
this.config.trialCommand,
nniManagerIp,
this.remoteRestServerPort,
version,
this.logCollection, cudaVisible);
this.logCollection,
cudaVisible);
//create tmp trial working folder locally.
await execMkdir(path.join(trialLocalTempFolder, '.nni'));
......
ts/nni_manager/training_service/remote_machine/shellExecutor.ts
View file @ 817ec68b
......@@ -44,24 +44,24 @@ class ShellExecutor {
const deferred: Deferred<void> = new Deferred<void>();
const connectConfig: ConnectConfig = {
host: rmMeta.ip,
port: rmMeta.port,
username: rmMeta.username,
host: rmMeta.config.host,
port: rmMeta.config.port,
username: rmMeta.config.user,
tryKeyboard: true,
};
this.pythonPath = rmMeta.pythonPath;
this.name = `${rmMeta.username}@${rmMeta.ip}:${rmMeta.port}`;
if (rmMeta.passwd !== undefined) {
connectConfig.password = rmMeta.passwd;
} else if (rmMeta.sshKeyPath !== undefined) {
if (!fs.existsSync(rmMeta.sshKeyPath)) {
this.pythonPath = rmMeta.config.pythonPath;
this.name = `${rmMeta.config.user}@${rmMeta.config.host}:${rmMeta.config.port}`;
if (rmMeta.config.password !== undefined) {
connectConfig.password = rmMeta.config.password;
} else if (rmMeta.config.sshKeyFile !== undefined) {
if (!fs.existsSync(rmMeta.config.sshKeyFile)) {
//SSh key path is not a valid file, reject
deferred.reject(new Error(`${rmMeta.sshKeyPath} does not exist.`));
deferred.reject(new Error(`${rmMeta.config.sshKeyFile} does not exist.`));
}
const privateKey: string = fs.readFileSync(rmMeta.sshKeyPath, 'utf8');
const privateKey: string = fs.readFileSync(rmMeta.config.sshKeyFile, 'utf8');
connectConfig.privateKey = privateKey;
connectConfig.passphrase = rmMeta.passphrase;
connectConfig.passphrase = rmMeta.config.sshPassphrase;
} else {
deferred.reject(new Error(`No valid passwd or sshKeyPath is configed.`));
}
......@@ -101,7 +101,7 @@ class ShellExecutor {
// SSH connection error, reject with error message
deferred.reject(new Error(err.message));
}).on("keyboard-interactive", (_name, _instructions, _lang, _prompts, finish) => {
finish([rmMeta.passwd]);
finish([rmMeta.config.password || '']);
}).connect(connectConfig);
return deferred.promise;
......
ts/nni_manager/training_service/reusable/environment.ts
View file @ 817ec68b
......@@ -129,7 +129,6 @@ export class EnvironmentInformation {
export abstract class EnvironmentService {
public abstract get hasStorageService(): boolean;
public abstract config(key: string, value: string): Promise<void>;
public abstract refreshEnvironmentsStatus(environments: EnvironmentInformation[]): Promise<void>;
public abstract stopEnvironment(environment: EnvironmentInformation): Promise<void>;
public abstract startEnvironment(environment: EnvironmentInformation): Promise<void>;
......
ts/nni_manager/training_service/reusable/environments/environmentServiceFactory.ts
View file @ 817ec68b
......@@ -3,18 +3,19 @@ import { OpenPaiEnvironmentService } from './openPaiEnvironmentService';
import { LocalEnvironmentService } from './localEnvironmentService';
import { RemoteEnvironmentService } from './remoteEnvironmentService';
import { EnvironmentService } from '../environment';
import { ExperimentConfig } from '../../../common/experimentConfig';
export class EnvironmentServiceFactory {
public static createEnvironmentService(name: string): EnvironmentService {
public static createEnvironmentService(name: string, config: ExperimentConfig): EnvironmentService {
switch(name) {
case 'local':
return new LocalEnvironmentService();
return new LocalEnvironmentService(config);
case 'remote':
return new RemoteEnvironmentService();
return new RemoteEnvironmentService(config);
case 'aml':
return new AMLEnvironmentService();
case 'pai':
return new OpenPaiEnvironmentService();
case 'openpai':
return new OpenPaiEnvironmentService(config);
default:
throw new Error(`${name} not supported!`);
}
......
ts/nni_manager/training_service/reusable/environments/localEnvironmentService.ts
View file @ 817ec68b
......@@ -9,9 +9,8 @@ import * as tkill from 'tree-kill';
import * as component from '../../../common/component';
import { getExperimentId } from '../../../common/experimentStartupInfo';
import { getLogger, Logger } from '../../../common/log';
import { TrialConfigMetadataKey } from '../../common/trialConfigMetadataKey';
import { ExperimentConfig } from '../../../common/experimentConfig';
import { EnvironmentInformation, EnvironmentService } from '../environment';
import { TrialConfig } from '../../common/trialConfig';
import { getExperimentRootDir, isAlive, getNewLine } from '../../../common/utils';
import { execMkdir, runScript, getScriptName, execCopydir } from '../../common/util';
import { SharedStorageService } from '../sharedStorage'
......@@ -20,11 +19,10 @@ import { SharedStorageService } from '../sharedStorage'
export class LocalEnvironmentService extends EnvironmentService {
private readonly log: Logger = getLogger();
private localTrialConfig: TrialConfig | undefined;
private experimentRootDir: string;
private experimentId: string;
constructor() {
constructor(_config: ExperimentConfig) {
super();
this.experimentId = getExperimentId();
this.experimentRootDir = getExperimentRootDir();
......@@ -42,16 +40,6 @@ export class LocalEnvironmentService extends EnvironmentService {
return 'local';
}
public async config(key: string, value: string): Promise<void> {
switch (key) {
case TrialConfigMetadataKey.TRIAL_CONFIG:
this.localTrialConfig = <TrialConfig>JSON.parse(value);
break;
default:
this.log.debug(`Local mode does not proccess metadata key: '${key}', value: '${value}'`);
}
}
public async refreshEnvironmentsStatus(environments: EnvironmentInformation[]): Promise<void> {
environments.forEach(async (environment) => {
const jobpidPath: string = `${path.join(environment.runnerWorkingFolder, 'pid')}`;
......@@ -118,9 +106,6 @@ export class LocalEnvironmentService extends EnvironmentService {
}
public async startEnvironment(environment: EnvironmentInformation): Promise<void> {
if (this.localTrialConfig === undefined) {
throw new Error('Local trial config is not initialized');
}
// Need refactor, this temp folder path is not appropriate, there are two expId in this path
const sharedStorageService = component.get<SharedStorageService>(SharedStorageService);
if (environment.useSharedStorage && sharedStorageService.canLocalMounted) {
......
ts/nni_manager/training_service/reusable/environments/openPaiEnvironmentService.ts
View file @ 817ec68b
......@@ -3,20 +3,20 @@
'use strict';
import * as fs from 'fs';
import * as yaml from 'js-yaml';
import * as request from 'request';
import { Deferred } from 'ts-deferred';
import * as component from '../../../common/component';
import { getExperimentId } from '../../../common/experimentStartupInfo';
import { ExperimentConfig, OpenpaiConfig, flattenConfig, toMegaBytes } from '../../../common/experimentConfig';
import { getLogger, Logger } from '../../../common/log';
import { TrialConfigMetadataKey } from '../../common/trialConfigMetadataKey';
import { PAIClusterConfig } from '../../pai/paiConfig';
import { NNIPAITrialConfig } from '../../pai/paiConfig';
import { EnvironmentInformation, EnvironmentService } from '../environment';
import { SharedStorageService } from '../sharedStorage';
import { StorageService } from '../storageService';
import { MountedStorageService } from '../storages/mountedStorageService';
interface FlattenOpenpaiConfig extends ExperimentConfig, OpenpaiConfig { }
/**
* Collector PAI jobs info from PAI cluster, and update pai job status locally
......@@ -27,15 +27,22 @@ export class OpenPaiEnvironmentService extends EnvironmentService {
private readonly log: Logger = getLogger();
private paiClusterConfig: PAIClusterConfig | undefined;
private paiTrialConfig: NNIPAITrialConfig | undefined;
private paiJobConfig: any;
private paiToken?: string;
private protocol: string = 'http';
private paiToken: string;
private protocol: string;
private experimentId: string;
private config: FlattenOpenpaiConfig;
constructor() {
constructor(config: ExperimentConfig) {
super();
this.experimentId = getExperimentId();
this.config = flattenConfig(config, 'openpai');
this.paiToken = this.config.token;
this.protocol = this.config.host.toLowerCase().startsWith('https://') ? 'https' : 'http';
// FIXME: only support MountedStorageService
const storageService = new MountedStorageService();
const remoteRoot = storageService.joinPath(this.config.localStorageMountPoint, this.experimentId);
storageService.initialize(this.config.localStorageMountPoint, remoteRoot);
}
public get environmentMaintenceLoopInterval(): number {
......@@ -50,58 +57,15 @@ export class OpenPaiEnvironmentService extends EnvironmentService {
return 'pai';
}
public async config(key: string, value: string): Promise<void> {
switch (key) {
case TrialConfigMetadataKey.PAI_CLUSTER_CONFIG:
this.paiClusterConfig = <PAIClusterConfig>JSON.parse(value);
this.paiClusterConfig.host = this.formatPAIHost(this.paiClusterConfig.host);
this.paiToken = this.paiClusterConfig.token;
break;
case TrialConfigMetadataKey.TRIAL_CONFIG: {
if (this.paiClusterConfig === undefined) {
this.log.error('pai cluster config is not initialized');
break;
}
this.paiTrialConfig = <NNIPAITrialConfig>JSON.parse(value);
// Validate to make sure codeDir doesn't have too many files
const storageService = component.get<StorageService>(StorageService);
const remoteRoot = storageService.joinPath(this.paiTrialConfig.nniManagerNFSMountPath, this.experimentId);
storageService.initialize(this.paiTrialConfig.nniManagerNFSMountPath, remoteRoot);
if (this.paiTrialConfig.paiConfigPath) {
this.paiJobConfig = yaml.safeLoad(fs.readFileSync(this.paiTrialConfig.paiConfigPath, 'utf8'));
}
if (this.paiClusterConfig.gpuNum === undefined) {
this.paiClusterConfig.gpuNum = this.paiTrialConfig.gpuNum;
}
if (this.paiClusterConfig.cpuNum === undefined) {
this.paiClusterConfig.cpuNum = this.paiTrialConfig.cpuNum;
}
if (this.paiClusterConfig.memoryMB === undefined) {
this.paiClusterConfig.memoryMB = this.paiTrialConfig.memoryMB;
}
break;
}
default:
this.log.debug(`OpenPAI not proccessed metadata key: '${key}', value: '${value}'`);
}
}
public async refreshEnvironmentsStatus(environments: EnvironmentInformation[]): Promise<void> {
const deferred: Deferred<void> = new Deferred<void>();
if (this.paiClusterConfig === undefined) {
throw new Error('PAI Cluster config is not initialized');
}
if (this.paiToken === undefined) {
throw new Error('PAI token is not initialized');
}
const getJobInfoRequest: request.Options = {
uri: `${this.protocol}://${this.paiClusterConfig.host}/rest-server/api/v2/jobs?username=${this.paiClusterConfig.userName}`,
uri: `${this.config.host}/rest-server/api/v2/jobs?username=${this.config.username}`,
method: 'GET',
json: true,
headers: {
......@@ -168,29 +132,22 @@ export class OpenPaiEnvironmentService extends EnvironmentService {
public async startEnvironment(environment: EnvironmentInformation): Promise<void> {
const deferred: Deferred<void> = new Deferred<void>();
if (this.paiClusterConfig === undefined) {
throw new Error('PAI Cluster config is not initialized');
}
if (this.paiToken === undefined) {
throw new Error('PAI token is not initialized');
}
if (this.paiTrialConfig === undefined) {
throw new Error('PAI trial config is not initialized');
}
// Step 1. Prepare PAI job configuration
let environmentRoot: string;
if (environment.useSharedStorage) {
environmentRoot = component.get<SharedStorageService>(SharedStorageService).remoteWorkingRoot;
environment.command = `${component.get<SharedStorageService>(SharedStorageService).remoteMountCommand.replace(/echo -e /g, `echo `).replace(/echo /g, `echo -e `)} && cd ${environmentRoot} && ${environment.command}`;
} else {
environmentRoot = `${this.paiTrialConfig.containerNFSMountPath}/${this.experimentId}`;
environmentRoot = `${this.config.containerStorageMountPoint}/${this.experimentId}`;
environment.command = `cd ${environmentRoot} && ${environment.command}`;
}
environment.runnerWorkingFolder = `${environmentRoot}/envs/${environment.id}`;
environment.trackingUrl = `${this.protocol}://${this.paiClusterConfig.host}/job-detail.html?username=${this.paiClusterConfig.userName}&jobName=${environment.envId}`;
environment.useActiveGpu = this.paiClusterConfig.useActiveGpu;
environment.maxTrialNumberPerGpu = this.paiClusterConfig.maxTrialNumPerGpu;
environment.trackingUrl = `${this.config.host}/job-detail.html?username=${this.config.username}&jobName=${environment.envId}`;
environment.useActiveGpu = false; // does openpai supports these?
environment.maxTrialNumberPerGpu = 1;
// Step 2. Generate Job Configuration in yaml format
const paiJobConfig = this.generateJobConfigInYamlFormat(environment);
......@@ -198,7 +155,7 @@ export class OpenPaiEnvironmentService extends EnvironmentService {
// Step 3. Submit PAI job via Rest call
const submitJobRequest: request.Options = {
uri: `${this.protocol}://${this.paiClusterConfig.host}/rest-server/api/v2/jobs`,
uri: `${this.config.host}/rest-server/api/v2/jobs`,
method: 'POST',
body: paiJobConfig,
followAllRedirects: true,
......@@ -229,15 +186,12 @@ export class OpenPaiEnvironmentService extends EnvironmentService {
if (environment.isAlive === false) {
return Promise.resolve();
}
if (this.paiClusterConfig === undefined) {
return Promise.reject(new Error('PAI Cluster config is not initialized'));
}
if (this.paiToken === undefined) {
return Promise.reject(Error('PAI token is not initialized'));
}
const stopJobRequest: request.Options = {
uri: `${this.protocol}://${this.paiClusterConfig.host}/rest-server/api/v2/jobs/${this.paiClusterConfig.userName}~${environment.envId}/executionType`,
uri: `${this.config.host}/rest-server/api/v2/jobs/${this.config.username}~${environment.envId}/executionType`,
method: 'PUT',
json: true,
body: { value: 'STOP' },
......@@ -278,14 +232,11 @@ export class OpenPaiEnvironmentService extends EnvironmentService {
}
private generateJobConfigInYamlFormat(environment: EnvironmentInformation): any {
if (this.paiTrialConfig === undefined) {
throw new Error('trial config is not initialized');
}
const jobName = environment.envId;
let nniJobConfig: any = undefined;
if (this.paiTrialConfig.paiConfigPath) {
nniJobConfig = JSON.parse(JSON.stringify(this.paiJobConfig)); //Trick for deep clone in Typescript
if (this.config.openpaiConfig !== undefined) {
nniJobConfig = JSON.parse(JSON.stringify(this.config.openpaiConfig)); //Trick for deep clone in Typescript
nniJobConfig.name = jobName;
if (nniJobConfig.taskRoles) {
......@@ -313,19 +264,6 @@ export class OpenPaiEnvironmentService extends EnvironmentService {
}
} else {
if (this.paiClusterConfig === undefined) {
throw new Error('PAI Cluster config is not initialized');
}
if (this.paiClusterConfig.gpuNum === undefined) {
throw new Error('PAI Cluster gpuNum is not initialized');
}
if (this.paiClusterConfig.cpuNum === undefined) {
throw new Error('PAI Cluster cpuNum is not initialized');
}
if (this.paiClusterConfig.memoryMB === undefined) {
throw new Error('PAI Cluster memoryMB is not initialized');
}
nniJobConfig = {
protocolVersion: 2,
name: jobName,
......@@ -334,7 +272,7 @@ export class OpenPaiEnvironmentService extends EnvironmentService {
prerequisites: [
{
type: 'dockerimage',
uri: this.paiTrialConfig.image,
uri: this.config.dockerImage,
name: 'docker_image_0'
}
],
......@@ -348,9 +286,9 @@ export class OpenPaiEnvironmentService extends EnvironmentService {
taskRetryCount: 0,
dockerImage: 'docker_image_0',
resourcePerInstance: {
gpu: this.paiClusterConfig.gpuNum,
cpu: this.paiClusterConfig.cpuNum,
memoryMB: this.paiClusterConfig.memoryMB
gpu: this.config.trialGpuNumber,
cpu: this.config.trialCpuNumber,
memoryMB: toMegaBytes(this.config.trialMemorySize)
},
commands: [
environment.command
......@@ -360,15 +298,15 @@ export class OpenPaiEnvironmentService extends EnvironmentService {
extras: {
'storages': [
{
name: this.paiTrialConfig.paiStorageConfigName
name: this.config.storageConfigName
}
],
submitFrom: 'submit-job-v2'
}
}
if (this.paiTrialConfig.virtualCluster) {
if (this.config.deprecated && this.config.deprecated.virtualCluster) {
nniJobConfig.defaults = {
virtualCluster: this.paiTrialConfig.virtualCluster
virtualCluster: this.config.deprecated.virtualCluster
}
}
}
......
ts/nni_manager/training_service/reusable/environments/remoteEnvironmentService.ts
View file @ 817ec68b
......@@ -9,44 +9,50 @@ import * as component from '../../../common/component';
import { getExperimentId } from '../../../common/experimentStartupInfo';
import { getLogger, Logger } from '../../../common/log';
import { EnvironmentInformation, EnvironmentService } from '../environment';
import {
getExperimentRootDir, getLogLevel
} from '../../../common/utils';
import { TrialConfig } from '../../common/trialConfig';
import { TrialConfigMetadataKey } from '../../common/trialConfigMetadataKey';
import { execMkdir, validateCodeDir } from '../../common/util';
import {
ExecutorManager, RemoteMachineMeta,
} from '../../remote_machine/remoteMachineData';
import { getExperimentRootDir, getLogLevel } from '../../../common/utils';
import { ExperimentConfig, RemoteConfig, RemoteMachineConfig, flattenConfig } from '../../../common/experimentConfig';
import { execMkdir } from '../../common/util';
import { ExecutorManager } from '../../remote_machine/remoteMachineData';
import { ShellExecutor } from 'training_service/remote_machine/shellExecutor';
import { RemoteMachineEnvironmentInformation } from '../remote/remoteConfig';
import { SharedStorageService } from '../sharedStorage'
interface FlattenRemoteConfig extends ExperimentConfig, RemoteConfig { }
@component.Singleton
export class RemoteEnvironmentService extends EnvironmentService {
private readonly initExecutorId = "initConnection";
private readonly machineExecutorManagerMap: Map<RemoteMachineMeta, ExecutorManager>;
private readonly machineExecutorManagerMap: Map<RemoteMachineConfig, ExecutorManager>;
private readonly environmentExecutorManagerMap: Map<string, ExecutorManager>;
private readonly remoteMachineMetaOccupiedMap: Map<RemoteMachineMeta, boolean>;
private trialConfig: TrialConfig | undefined;
private readonly remoteMachineMetaOccupiedMap: Map<RemoteMachineConfig, boolean>;
private readonly log: Logger;
private sshConnectionPromises: any[];
private experimentRootDir: string;
private remoteExperimentRootDir: string = "";
private experimentId: string;
private config: FlattenRemoteConfig;
constructor() {
constructor(config: ExperimentConfig) {
super();
this.experimentId = getExperimentId();
this.environmentExecutorManagerMap = new Map<string, ExecutorManager>();
this.machineExecutorManagerMap = new Map<RemoteMachineMeta, ExecutorManager>();
this.remoteMachineMetaOccupiedMap = new Map<RemoteMachineMeta, boolean>();
this.machineExecutorManagerMap = new Map<RemoteMachineConfig, ExecutorManager>();
this.remoteMachineMetaOccupiedMap = new Map<RemoteMachineConfig, boolean>();
this.sshConnectionPromises = [];
this.experimentRootDir = getExperimentRootDir();
this.experimentId = getExperimentId();
this.log = getLogger();
this.config = flattenConfig(config, 'remote');
// codeDir is not a valid directory, throw Error
if (!fs.lstatSync(this.config.trialCodeDirectory).isDirectory()) {
throw new Error(`codeDir ${this.config.trialCodeDirectory} is not a directory`);
}
this.sshConnectionPromises = this.config.machineList.map(
machine => this.initRemoteMachineOnConnected(machine)
);
}
public get prefetchedEnvironmentCount(): number {
......@@ -69,39 +75,7 @@ export class RemoteEnvironmentService extends EnvironmentService {
return 'remote';
}
public async config(key: string, value: string): Promise<void> {
switch (key) {
case TrialConfigMetadataKey.MACHINE_LIST:
await this.setupConnections(value);
break;
case TrialConfigMetadataKey.TRIAL_CONFIG: {
const remoteMachineTrailConfig: TrialConfig = <TrialConfig>JSON.parse(value);
// Parse trial config failed, throw Error
if (remoteMachineTrailConfig === undefined) {
throw new Error('trial config parsed failed');
}
// codeDir is not a valid directory, throw Error
if (!fs.lstatSync(remoteMachineTrailConfig.codeDir)
.isDirectory()) {
throw new Error(`codeDir ${remoteMachineTrailConfig.codeDir} is not a directory`);
}
try {
// Validate to make sure codeDir doesn't have too many files
await validateCodeDir(remoteMachineTrailConfig.codeDir);
} catch (error) {
this.log.error(error);
return Promise.reject(new Error(error));
}
this.trialConfig = remoteMachineTrailConfig;
break;
}
default:
this.log.debug(`Remote not support metadata key: '${key}', value: '${value}'`);
}
}
private scheduleMachine(): RemoteMachineMeta | undefined {
private scheduleMachine(): RemoteMachineConfig | undefined {
for (const [rmMeta, occupied] of this.remoteMachineMetaOccupiedMap) {
if (!occupied) {
this.remoteMachineMetaOccupiedMap.set(rmMeta, true);
......@@ -111,19 +85,9 @@ export class RemoteEnvironmentService extends EnvironmentService {
return undefined;
}
private async setupConnections(machineList: string): Promise<void> {
this.log.debug(`Connecting to remote machines: ${machineList}`);
//TO DO: verify if value's format is wrong, and json parse failed, how to handle error
const rmMetaList: RemoteMachineMeta[] = <RemoteMachineMeta[]>JSON.parse(machineList);
for (const rmMeta of rmMetaList) {
this.sshConnectionPromises.push(await this.initRemoteMachineOnConnected(rmMeta));
}
}
private async initRemoteMachineOnConnected(rmMeta: RemoteMachineMeta): Promise<void> {
private async initRemoteMachineOnConnected(rmMeta: RemoteMachineConfig): Promise<void> {
const executorManager: ExecutorManager = new ExecutorManager(rmMeta);
this.log.info(`connecting to ${rmMeta.username}@${rmMeta.ip}:${rmMeta.port}`);
this.log.info(`connecting to ${rmMeta.user}@${rmMeta.host}:${rmMeta.port}`);
const executor: ShellExecutor = await executorManager.getExecutor(this.initExecutorId);
this.log.debug(`reached ${executor.name}`);
this.machineExecutorManagerMap.set(rmMeta, executorManager);
......@@ -142,10 +106,7 @@ export class RemoteEnvironmentService extends EnvironmentService {
}
public async refreshEnvironmentsStatus(environments: EnvironmentInformation[]): Promise<void> {
const tasks: Promise<void>[] = [];
environments.forEach(async (environment) => {
tasks.push(this.refreshEnvironment(environment));
});
const tasks = environments.map(environment => this.refreshEnvironment(environment));
await Promise.all(tasks);
}
......@@ -168,7 +129,7 @@ export class RemoteEnvironmentService extends EnvironmentService {
if (remoteEnvironment.rmMachineMeta === undefined) {
throw new Error(`${remoteEnvironment.id} machine meta not initialized!`);
}
this.log.info(`pid in ${remoteEnvironment.rmMachineMeta.ip}:${jobpidPath} is not alive!`);
this.log.info(`pid in ${remoteEnvironment.rmMachineMeta.host}:${jobpidPath} is not alive!`);
if (fs.existsSync(runnerReturnCodeFilePath)) {
const runnerReturnCode: string = await executor.getRemoteFileContent(runnerReturnCodeFilePath);
const match: RegExpMatchArray | null = runnerReturnCode.trim()
......@@ -248,9 +209,6 @@ export class RemoteEnvironmentService extends EnvironmentService {
this.log.info('ssh connection initialized!');
// set sshConnectionPromises to [] to avoid log information duplicated
this.sshConnectionPromises = [];
if (this.trialConfig === undefined) {
throw new Error("trial config not initialized!");
}
Array.from(this.machineExecutorManagerMap.keys()).forEach(rmMeta => {
// initialize remoteMachineMetaOccupiedMap, false means not occupied
this.remoteMachineMetaOccupiedMap.set(rmMeta, false);
......@@ -265,12 +223,8 @@ export class RemoteEnvironmentService extends EnvironmentService {
}
private async prepareEnvironment(environment: RemoteMachineEnvironmentInformation): Promise<boolean> {
if (this.trialConfig === undefined) {
throw new Error('trial config is not initialized');
}
// get an executor from scheduler
const rmMachineMeta: RemoteMachineMeta | undefined = this.scheduleMachine();
const rmMachineMeta: RemoteMachineConfig | undefined = this.scheduleMachine();
if (rmMachineMeta === undefined) {
this.log.warning(`No available machine!`);
return Promise.resolve(false);
......@@ -299,9 +253,6 @@ export class RemoteEnvironmentService extends EnvironmentService {
}
private async launchRunner(environment: RemoteMachineEnvironmentInformation): Promise<void> {
if (this.trialConfig === undefined) {
throw new Error('trial config is not initialized');
}
const executor = await this.getExecutor(environment.id);
const environmentLocalTempFolder: string =
path.join(this.experimentRootDir, "environment-temp")
......@@ -317,7 +268,7 @@ export class RemoteEnvironmentService extends EnvironmentService {
if (environment.rmMachineMeta === undefined) {
throw new Error(`${environment.id} rmMachineMeta not initialized!`);
}
environment.trackingUrl = `file://${environment.rmMachineMeta.ip}:${environment.runnerWorkingFolder}`;
environment.trackingUrl = `file://${environment.rmMachineMeta.host}:${environment.runnerWorkingFolder}`;
}
private async getExecutor(environmentId: string): Promise<ShellExecutor> {
......@@ -330,7 +281,7 @@ export class RemoteEnvironmentService extends EnvironmentService {
public async stopEnvironment(environment: EnvironmentInformation): Promise<void> {
if (environment.isAlive === false) {
return Promise.resolve();
return;
}
const executor = await this.getExecutor(environment.id);
......@@ -338,7 +289,7 @@ export class RemoteEnvironmentService extends EnvironmentService {
if (environment.status === 'UNKNOWN') {
environment.status = 'USER_CANCELED';
await this.releaseEnvironmentResource(environment);
return
return;
}
const jobpidPath: string = `${environment.runnerWorkingFolder}/pid`;
......
ts/nni_manager/training_service/reusable/remote/remoteConfig.ts
View file @ 817ec68b
......@@ -2,23 +2,11 @@
// Licensed under the MIT license.
import { EnvironmentInformation } from '../environment';
import { RemoteMachineMeta } from '../../remote_machine/remoteMachineData';
import { RemoteMachineConfig } from '../../../common/experimentConfig';
/**
* RemoteMachineEnvironmentInformation
*/
export class RemoteMachineEnvironmentInformation extends EnvironmentInformation {
public rmMachineMeta?: RemoteMachineMeta;
}
export class RemoteConfig {
public readonly reuse: boolean;
/**
* Constructor
* @param reuse If job is reusable for multiple trials
*/
constructor(reuse: boolean) {
this.reuse = reuse;
}
public rmMachineMeta?: RemoteMachineConfig;
}
ts/nni_manager/training_service/reusable/routerTrainingService.ts
View file @ 817ec68b
......@@ -3,21 +3,15 @@
'use strict';
import { Container, Scope } from 'typescript-ioc';
import * as component from '../../common/component';
import { getLogger, Logger } from '../../common/log';
import { MethodNotImplementedError } from '../../common/errors'
import { MethodNotImplementedError } from '../../common/errors';
import { ExperimentConfig, RemoteConfig, OpenpaiConfig } from '../../common/experimentConfig';
import { TrainingService, TrialJobApplicationForm, TrialJobDetail, TrialJobMetric, LogType } from '../../common/trainingService';
import { delay } from '../../common/utils';
import { TrialConfigMetadataKey } from '../common/trialConfigMetadataKey';
import { PAIClusterConfig } from '../pai/paiConfig';
import { PAITrainingService } from '../pai/paiTrainingService';
import { RemoteMachineTrainingService } from '../remote_machine/remoteMachineTrainingService';
import { MountedStorageService } from './storages/mountedStorageService';
import { StorageService } from './storageService';
import { TrialDispatcher } from './trialDispatcher';
import { RemoteConfig } from './remote/remoteConfig';
import { HeterogenousConfig } from './heterogenous/heterogenousConfig';
/**
......@@ -26,11 +20,19 @@ import { HeterogenousConfig } from './heterogenous/heterogenousConfig';
*/
@component.Singleton
class RouterTrainingService implements TrainingService {
protected readonly log!: Logger;
private internalTrainingService: TrainingService | undefined;
protected readonly log: Logger;
private internalTrainingService: TrainingService;
constructor() {
constructor(config: ExperimentConfig) {
this.log = getLogger();
const platform = Array.isArray(config.trainingService) ? 'hybrid' : config.trainingService.platform;
if (platform === 'remote' && !(<RemoteConfig>config.trainingService).reuseMode) {
this.internalTrainingService = new RemoteMachineTrainingService(config);
} else if (platform === 'openpai' && !(<OpenpaiConfig>config.trainingService).reuseMode) {
this.internalTrainingService = new PAITrainingService(config);
} else {
this.internalTrainingService = new TrialDispatcher(config);
}
}
public async listTrialJobs(): Promise<TrialJobDetail[]> {
......@@ -79,13 +81,6 @@ class RouterTrainingService implements TrainingService {
return await this.internalTrainingService.updateTrialJob(trialJobId, form);
}
public get isMultiPhaseJobSupported(): boolean {
if (this.internalTrainingService === undefined) {
throw new Error("TrainingService is not assigned!");
}
return this.internalTrainingService.isMultiPhaseJobSupported;
}
public async cancelTrialJob(trialJobId: string, isEarlyStopped?: boolean | undefined): Promise<void> {
if (this.internalTrainingService === undefined) {
throw new Error("TrainingService is not assigned!");
......@@ -93,80 +88,8 @@ class RouterTrainingService implements TrainingService {
await this.internalTrainingService.cancelTrialJob(trialJobId, isEarlyStopped);
}
public async setClusterMetadata(key: string, value: string): Promise<void> {
if (this.internalTrainingService === undefined) {
// Need to refactor configuration, remove hybrid_config field in the future
if (key === TrialConfigMetadataKey.HYBRID_CONFIG){
this.internalTrainingService = component.get(TrialDispatcher);
const heterogenousConfig: HeterogenousConfig = <HeterogenousConfig>JSON.parse(value);
if (this.internalTrainingService === undefined) {
throw new Error("internalTrainingService not initialized!");
}
// Initialize storageService for pai, only support singleton for now, need refactor
if (heterogenousConfig.trainingServicePlatforms.includes('pai')) {
Container.bind(StorageService)
.to(MountedStorageService)
.scope(Scope.Singleton);
}
await this.internalTrainingService.setClusterMetadata('platform_list',
heterogenousConfig.trainingServicePlatforms.join(','));
} else if (key === TrialConfigMetadataKey.LOCAL_CONFIG) {
this.internalTrainingService = component.get(TrialDispatcher);
if (this.internalTrainingService === undefined) {
throw new Error("internalTrainingService not initialized!");
}
await this.internalTrainingService.setClusterMetadata('platform_list', 'local');
} else if (key === TrialConfigMetadataKey.PAI_CLUSTER_CONFIG) {
const config = <PAIClusterConfig>JSON.parse(value);
if (config.reuse === true) {
this.log.info(`reuse flag enabled, use EnvironmentManager.`);
this.internalTrainingService = component.get(TrialDispatcher);
// TODO to support other storages later.
Container.bind(StorageService)
.to(MountedStorageService)
.scope(Scope.Singleton);
if (this.internalTrainingService === undefined) {
throw new Error("internalTrainingService not initialized!");
}
await this.internalTrainingService.setClusterMetadata('platform_list', 'pai');
} else {
this.log.debug(`caching metadata key:{} value:{}, as training service is not determined.`);
this.internalTrainingService = component.get(PAITrainingService);
}
} else if (key === TrialConfigMetadataKey.AML_CLUSTER_CONFIG) {
this.internalTrainingService = component.get(TrialDispatcher);
if (this.internalTrainingService === undefined) {
throw new Error("internalTrainingService not initialized!");
}
await this.internalTrainingService.setClusterMetadata('platform_list', 'aml');
} else if (key === TrialConfigMetadataKey.REMOTE_CONFIG) {
const config = <RemoteConfig>JSON.parse(value);
if (config.reuse === true) {
this.log.info(`reuse flag enabled, use EnvironmentManager.`);
this.internalTrainingService = component.get(TrialDispatcher);
if (this.internalTrainingService === undefined) {
throw new Error("internalTrainingService not initialized!");
}
await this.internalTrainingService.setClusterMetadata('platform_list', 'remote');
} else {
this.log.debug(`caching metadata key:{} value:{}, as training service is not determined.`);
this.internalTrainingService = component.get(RemoteMachineTrainingService);
}
}
}
if (this.internalTrainingService === undefined) {
throw new Error("internalTrainingService not initialized!");
}
await this.internalTrainingService.setClusterMetadata(key, value);
}
public async getClusterMetadata(key: string): Promise<string> {
if (this.internalTrainingService === undefined) {
throw new Error("TrainingService is not assigned!");
}
return await this.internalTrainingService.getClusterMetadata(key);
}
public async setClusterMetadata(_key: string, _value: string): Promise<void> { return; }
public async getClusterMetadata(_key: string): Promise<string> { return ''; }
public async cleanUp(): Promise<void> {
if (this.internalTrainingService === undefined) {
......
ts/nni_manager/training_service/reusable/sharedStorage.ts
View file @ 817ec68b
......@@ -3,19 +3,14 @@
'use strict';
import { SharedStorageConfig } from '../../common/experimentConfig';
import { StorageService } from './storageService'
export type SharedStorageType = 'NFS' | 'AzureBlob'
export type LocalMountedType = 'usermount' | 'nnimount' | 'nomount'
export interface SharedStorageConfig {
readonly storageType: SharedStorageType;
readonly localMountPoint?: string;
readonly remoteMountPoint: string;
}
export abstract class SharedStorageService {
public abstract config(key: string, value: string): Promise<void>;
public abstract config(config: SharedStorageConfig): Promise<void>;
public abstract get canLocalMounted(): boolean;
public abstract get storageService(): StorageService;
public abstract get localMountCommand(): string;
......
ts/nni_manager/training_service/reusable/shared_storages/azureblobStorageService.ts
View file @ 817ec68b
......@@ -6,11 +6,11 @@
import * as cpp from 'child-process-promise';
import * as path from 'path';
import { SharedStorageService, SharedStorageConfig, SharedStorageType, LocalMountedType } from '../sharedStorage'
import { SharedStorageService, SharedStorageType } from '../sharedStorage'
import { MountedStorageService } from '../storages/mountedStorageService';
import { TrialConfigMetadataKey } from '../../common/trialConfigMetadataKey';
import { getLogger, Logger } from '../../../common/log';
import { getExperimentId } from '../../../common/experimentStartupInfo';
import { AzureBlobConfig } from '../../../common/experimentConfig';
const INSTALL_BLOBFUSE = `
#!/bin/bash
......@@ -50,31 +50,6 @@ else
fi
`
class AzureBlobSharedStorageConfig implements SharedStorageConfig {
public storageType: SharedStorageType;
public localMountPoint?: string;
public remoteMountPoint: string;
public resourceGroupName?: string;
public storageAccountName: string;
public storageAccountKey?: string;
public containerName: string;
public localMounted: LocalMountedType;
constructor(storageType: SharedStorageType, remoteMountPoint: string, storageAccountName: string, containerName: string,
localMounted: LocalMountedType, localMountPoint?: string, resourceGroupName?: string, storageAccountKey?: string) {
this.storageType = storageType;
this.localMountPoint = localMountPoint;
this.remoteMountPoint = remoteMountPoint;
this.resourceGroupName = resourceGroupName;
this.storageAccountName = storageAccountName;
this.storageAccountKey = storageAccountKey;
this.containerName = containerName;
this.localMounted = localMounted;
}
}
export class AzureBlobSharedStorageService extends SharedStorageService {
private log: Logger;
private internalStorageService: MountedStorageService;
......@@ -96,13 +71,11 @@ export class AzureBlobSharedStorageService extends SharedStorageService {
this.experimentId = getExperimentId();
}
public async config(key: string, value: string): Promise<void> {
if (key === TrialConfigMetadataKey.SHARED_STORAGE_CONFIG) {
const azureblobConfig = <AzureBlobSharedStorageConfig>JSON.parse(value);
public async config(azureblobConfig: AzureBlobConfig): Promise<void> {
this.localMountPoint = azureblobConfig.localMountPoint;
this.remoteMountPoint = azureblobConfig.remoteMountPoint;
this.storageType = azureblobConfig.storageType;
this.storageType = azureblobConfig.storageType as SharedStorageType;
this.storageAccountName = azureblobConfig.storageAccountName;
this.containerName = azureblobConfig.containerName;
if (azureblobConfig.storageAccountKey !== undefined) {
......@@ -127,7 +100,6 @@ export class AzureBlobSharedStorageService extends SharedStorageService {
this.internalStorageService.initialize(this.localMountPoint, path.join(this.localMountPoint, 'nni', this.experimentId));
}
}
}
public get canLocalMounted(): boolean{
return true;
......
ts/nni_manager/training_service/reusable/shared_storages/nfsStorageService.ts
View file @ 817ec68b
......@@ -6,11 +6,11 @@
import * as cpp from 'child-process-promise';
import * as path from 'path';
import { SharedStorageService, SharedStorageConfig, SharedStorageType, LocalMountedType } from '../sharedStorage'
import { SharedStorageService, SharedStorageType } from '../sharedStorage'
import { MountedStorageService } from '../storages/mountedStorageService';
import { TrialConfigMetadataKey } from '../../common/trialConfigMetadataKey';
import { getLogger, Logger } from '../../../common/log';
import { getExperimentId } from '../../../common/experimentStartupInfo';
import { NfsConfig } from '../../../common/experimentConfig';
const INSTALL_NFS_CLIENT = `
#!/bin/bash
......@@ -35,26 +35,6 @@ else
fi
`
class NFSSharedStorageConfig implements SharedStorageConfig {
public storageType: SharedStorageType;
public localMountPoint: string;
public remoteMountPoint: string;
public nfsServer: string;
public exportedDirectory: string;
public localMounted: LocalMountedType;
constructor(storageType: SharedStorageType, localMountPoint: string, remoteMountPoint: string,
nfsServer: string, exportedDirectory: string, localMounted: LocalMountedType) {
this.storageType = storageType;
this.localMountPoint = localMountPoint;
this.remoteMountPoint = remoteMountPoint;
this.nfsServer = nfsServer;
this.exportedDirectory = exportedDirectory;
this.localMounted = localMounted;
}
}
export class NFSSharedStorageService extends SharedStorageService {
private log: Logger;
private internalStorageService: MountedStorageService;
......@@ -75,9 +55,7 @@ export class NFSSharedStorageService extends SharedStorageService {
this.experimentId = getExperimentId();
}
public async config(key: string, value: string): Promise<void> {
if (key === TrialConfigMetadataKey.SHARED_STORAGE_CONFIG) {
const nfsConfig = <NFSSharedStorageConfig>JSON.parse(value);
public async config(nfsConfig: NfsConfig): Promise<void> {
this.localMountPoint = nfsConfig.localMountPoint;
this.remoteMountPoint = nfsConfig.remoteMountPoint;
......@@ -94,7 +72,6 @@ export class NFSSharedStorageService extends SharedStorageService {
}
this.internalStorageService.initialize(this.localMountPoint, path.join(this.localMountPoint, 'nni', this.experimentId));
}
return Promise.resolve();
}
......
ts/nni_manager/training_service/reusable/test/trialDispatcher.test.ts
View file @ 817ec68b
......@@ -169,6 +169,18 @@ async function waitEnvironment(waitCount: number,
return waitRequestEnvironment;
}
const config = {
searchSpace: { },
trialCommand: 'echo hi',
trialCodeDirectory: path.dirname(__filename),
trialConcurrency: 0,
nniManagerIp: '127.0.0.1',
trainingService: {
platform: 'local'
},
debug: true
};
describe('Unit Test for TrialDispatcher', () => {
let trialRunPromise: Promise<void>;
......@@ -191,17 +203,8 @@ describe('Unit Test for TrialDispatcher', () => {
});
beforeEach(async () => {
const trialConfig = {
codeDir: currentDir,
command: "echo",
}
const nniManagerIpConfig = {
nniManagerIp: "127.0.0.1",
}
trialDispatcher = new TrialDispatcher();
trialDispatcher = new TrialDispatcher(config);
await trialDispatcher.setClusterMetadata(TrialConfigMetadataKey.TRIAL_CONFIG, JSON.stringify(trialConfig));
await trialDispatcher.setClusterMetadata(TrialConfigMetadataKey.NNI_MANAGER_IP, JSON.stringify(nniManagerIpConfig));
// set ut environment
let environmentServiceList: EnvironmentService[] = [];
environmentService = new UtEnvironmentService();
......@@ -224,7 +227,6 @@ describe('Unit Test for TrialDispatcher', () => {
});
it('reuse env', async () => {
let trialDetail = await newTrial(trialDispatcher);
await waitEnvironment(1, previousEnvironments, environmentService, commandChannel);
await verifyTrialRunning(commandChannel, trialDetail);
......@@ -240,31 +242,31 @@ describe('Unit Test for TrialDispatcher', () => {
});
it('not reusable env', async () => {
trialDispatcher.setClusterMetadata(
TrialConfigMetadataKey.TRIAL_CONFIG,
JSON.stringify({
reuseEnvironment: false,
codeDir: currentDir,
}));
//trialDispatcher.setClusterMetadata(
// TrialConfigMetadataKey.TRIAL_CONFIG,
// JSON.stringify({
// reuseEnvironment: false,
// codeDir: currentDir,
// }));
let trialDetail = await newTrial(trialDispatcher);
//let trialDetail = await newTrial(trialDispatcher);
let environment = await waitEnvironment(1, previousEnvironments, environmentService, commandChannel);
await verifyTrialRunning(commandChannel, trialDetail);
await verifyTrialResult(commandChannel, trialDetail, 0);
await waitResultMust<true>(async () => {
return environment.status === 'USER_CANCELED' ? true : undefined;
});
//let environment = await waitEnvironment(1, previousEnvironments, environmentService, commandChannel);
//await verifyTrialRunning(commandChannel, trialDetail);
//await verifyTrialResult(commandChannel, trialDetail, 0);
//await waitResultMust<true>(async () => {
// return environment.status === 'USER_CANCELED' ? true : undefined;
//});
trialDetail = await newTrial(trialDispatcher);
//trialDetail = await newTrial(trialDispatcher);
await waitEnvironment(2, previousEnvironments, environmentService, commandChannel);
await verifyTrialRunning(commandChannel, trialDetail);
await verifyTrialResult(commandChannel, trialDetail, -1);
//await waitEnvironment(2, previousEnvironments, environmentService, commandChannel);
//await verifyTrialRunning(commandChannel, trialDetail);
//await verifyTrialResult(commandChannel, trialDetail, -1);
chai.assert.equal(environmentService.testGetEnvironments().size, 2, "as env not reused, so only 2 envs should be here.");
const trials = await trialDispatcher.listTrialJobs();
chai.assert.equal(trials.length, 2, "there should be 2 trials");
//chai.assert.equal(environmentService.testGetEnvironments().size, 2, "as env not reused, so only 2 envs should be here.");
//const trials = await trialDispatcher.listTrialJobs();
//chai.assert.equal(trials.length, 2, "there should be 2 trials");
});
it('no more env', async () => {
......@@ -475,37 +477,37 @@ describe('Unit Test for TrialDispatcher', () => {
});
it('GPUScheduler disabled gpuNum === 0', async () => {
trialDispatcher.setClusterMetadata(
TrialConfigMetadataKey.TRIAL_CONFIG,
JSON.stringify({
reuseEnvironment: false,
codeDir: currentDir,
gpuNum: 0,
}));
//trialDispatcher.setClusterMetadata(
// TrialConfigMetadataKey.TRIAL_CONFIG,
// JSON.stringify({
// reuseEnvironment: false,
// codeDir: currentDir,
// gpuNum: 0,
// }));
let trialDetail = await newTrial(trialDispatcher);
await waitEnvironment(1, previousEnvironments, environmentService, commandChannel);
const command = await verifyTrialRunning(commandChannel, trialDetail);
await verifyTrialResult(commandChannel, trialDetail, 0);
//let trialDetail = await newTrial(trialDispatcher);
//await waitEnvironment(1, previousEnvironments, environmentService, commandChannel);
//const command = await verifyTrialRunning(commandChannel, trialDetail);
//await verifyTrialResult(commandChannel, trialDetail, 0);
chai.assert.equal(command.data["gpuIndices"], "");
//chai.assert.equal(command.data["gpuIndices"], "");
});
it('GPUScheduler enable no cluster gpu config', async () => {
trialDispatcher.setClusterMetadata(
TrialConfigMetadataKey.TRIAL_CONFIG,
JSON.stringify({
reuseEnvironment: false,
codeDir: currentDir,
gpuNum: 1,
}));
let trialDetail = await newTrial(trialDispatcher);
await waitEnvironment(1, previousEnvironments, environmentService, commandChannel);
const command = await verifyTrialRunning(commandChannel, trialDetail);
await verifyTrialResult(commandChannel, trialDetail, 0);
chai.assert.equal(command.data["gpuIndices"], "0");
//trialDispatcher.setClusterMetadata(
// TrialConfigMetadataKey.TRIAL_CONFIG,
// JSON.stringify({
// reuseEnvironment: false,
// codeDir: currentDir,
// gpuNum: 1,
// }));
//let trialDetail = await newTrial(trialDispatcher);
//await waitEnvironment(1, previousEnvironments, environmentService, commandChannel);
//const command = await verifyTrialRunning(commandChannel, trialDetail);
//await verifyTrialResult(commandChannel, trialDetail, 0);
//chai.assert.equal(command.data["gpuIndices"], "0");
});
it('GPUScheduler skipped no GPU info', async () => {
......
ts/nni_manager/training_service/test/localTrainingService.test.ts
View file @ 817ec68b
......@@ -13,6 +13,7 @@ import { TrialJobApplicationForm, TrialJobDetail} from '../../common/trainingSer
import { cleanupUnitTest, delay, prepareUnitTest, getExperimentRootDir } from '../../common/utils';
import { TrialConfigMetadataKey } from '../common/trialConfigMetadataKey';
import { LocalTrainingService } from '../local/localTrainingService';
import { ExperimentConfig } from '../../common/experimentConfig';
// TODO: copy mockedTrail.py to local folder
const localCodeDir: string = tmp.dirSync().name.split('\\').join('\\\\');
......@@ -20,9 +21,23 @@ const mockedTrialPath: string = './training_service/test/mockedTrial.py'
fs.copyFileSync(mockedTrialPath, localCodeDir + '/mockedTrial.py')
describe('Unit Test for LocalTrainingService', () => {
let trialConfig: any = `{"command":"sleep 1h && echo hello","codeDir":"${localCodeDir}","gpuNum":1}`
const config = <ExperimentConfig>{
trialCommand: 'sleep 1h && echo hello',
trialCodeDirectory: `${localCodeDir}`,
trialGpuNumber: 1,
trainingService: {
platform: 'local'
}
};
let localTrainingService: LocalTrainingService;
const config2 = <ExperimentConfig>{
trialCommand: 'python3 mockedTrial.py',
trialCodeDirectory: `${localCodeDir}`,
trialGpuNumber: 0,
trainingService: {
platform: 'local'
}
};
before(() => {
chai.should();
......@@ -34,29 +49,19 @@ describe('Unit Test for LocalTrainingService', () => {
cleanupUnitTest();
});
beforeEach(() => {
localTrainingService = component.get(LocalTrainingService);
it('List empty trial jobs', async () => {
const localTrainingService = new LocalTrainingService(config);
localTrainingService.run();
});
afterEach(() => {
localTrainingService.cleanUp();
});
it('List empty trial jobs', async () => {
//trial jobs should be empty, since there are no submitted jobs
chai.expect(await localTrainingService.listTrialJobs()).to.be.empty;
});
it('setClusterMetadata and getClusterMetadata', async () => {
await localTrainingService.setClusterMetadata(TrialConfigMetadataKey.TRIAL_CONFIG, trialConfig);
localTrainingService.getClusterMetadata(TrialConfigMetadataKey.TRIAL_CONFIG).then((data)=>{
chai.expect(data).to.be.equals(trialConfig);
});
localTrainingService.cleanUp();
});
it('Submit job and Cancel job', async () => {
await localTrainingService.setClusterMetadata(TrialConfigMetadataKey.TRIAL_CONFIG, trialConfig);
const localTrainingService = new LocalTrainingService(config);
localTrainingService.run();
// submit job
const form: TrialJobApplicationForm = {
......@@ -70,10 +75,13 @@ describe('Unit Test for LocalTrainingService', () => {
chai.expect(jobDetail.status).to.be.equals('WAITING');
await localTrainingService.cancelTrialJob(jobDetail.id);
chai.expect(jobDetail.status).to.be.equals('USER_CANCELED');
localTrainingService.cleanUp();
}).timeout(20000);
it('Get trial log', async () => {
await localTrainingService.setClusterMetadata(TrialConfigMetadataKey.TRIAL_CONFIG, trialConfig);
const localTrainingService = new LocalTrainingService(config);
localTrainingService.run();
// submit job
const form: TrialJobApplicationForm = {
......@@ -100,13 +108,14 @@ describe('Unit Test for LocalTrainingService', () => {
fs.rmdirSync(path.join(rootDir, 'trials'))
await localTrainingService.cancelTrialJob(jobDetail.id);
localTrainingService.cleanUp();
}).timeout(20000);
it('Read metrics, Add listener, and remove listener', async () => {
// set meta data
const trialConfig: string = `{\"command\":\"python3 mockedTrial.py\", \"codeDir\":\"${localCodeDir}\",\"gpuNum\":0}`
await localTrainingService.setClusterMetadata(TrialConfigMetadataKey.TRIAL_CONFIG, trialConfig);
const localTrainingService = new LocalTrainingService(config2);
localTrainingService.run();
// set meta data
// submit job
const form: TrialJobApplicationForm = {
sequenceId: 0,
......@@ -130,9 +139,6 @@ describe('Unit Test for LocalTrainingService', () => {
await localTrainingService.cancelTrialJob(jobDetail.id);
localTrainingService.removeTrialJobMetricListener(listener1);
localTrainingService.cleanUp();
}).timeout(20000);
it('Test multiphaseSupported', () => {
chai.expect(localTrainingService.isMultiPhaseJobSupported).to.be.equals(true)
})
});
ts/webui/mock/all-types-metric.json
View file @ 817ec68b
......@@ -11,30 +11,28 @@
"logDir": "/***/nni/experiments/Tkaxm2mb",
"nextSequenceId": 110,
"params": {
"authorName": "default",
"experimentName": "default",
"trialConcurrency": 10,
"maxExecDuration": 3600,
"maxTrialNum": 100,
"searchSpace": "{\"intermediate1\": {\"_type\": \"choice\", \"_value\": [\"normal\", \"inf\", \"neginf\", \"nan\", \"string\", \"dict-empty\", \"dict-normal\", \"dict-nodefault\", \"dict-defaultdict\"]}, \"intermediate2\": {\"_type\": \"choice\", \"_value\": [\"normal\", \"inf\", \"neginf\", \"nan\", \"string\", \"dict-empty\", \"dict-normal\", \"dict-nodefault\", \"dict-defaultdict\"]}, \"intermediate3\": {\"_type\": \"choice\", \"_value\": [\"normal\", \"inf\", \"neginf\", \"nan\", \"string\", \"dict-empty\", \"dict-normal\", \"dict-nodefault\", \"dict-defaultdict\"]}, \"intermediate_count\": {\"_type\": \"choice\", \"_value\": [0, 1, 2, 3]}, \"final1\": {\"_type\": \"choice\", \"_value\": [\"normal\", \"inf\", \"neginf\", \"nan\", \"string\", \"dict-empty\", \"dict-normal\", \"dict-nodefault\", \"dict-defaultdict\"]}, \"final2\": {\"_type\": \"choice\", \"_value\": [\"normal\", \"inf\", \"neginf\", \"nan\", \"string\", \"dict-empty\", \"dict-normal\", \"dict-nodefault\", \"dict-defaultdict\"]}, \"final_count\": {\"_type\": \"choice\", \"_value\": [0, 1, 2]}}",
"trainingServicePlatform": "local",
"tuner": {
"codeDir": "/***/nnidev/src/webui/tests/metrics-test/.",
"classFileName": "naive_random.py",
"className": "NaiveRandomTuner",
"checkpointDir": "/***/nni/experiments/Tkaxm2mb/checkpoint"
"maxExperimentDuration": "1h",
"maxTrialNumber": 100,
"searchSpace": {
"intermediate1": {"_type": "choice", "_value": [ "normal", "inf", "neginf", "nan", "string", "dict-empty", "dict-normal", "dict-nodefault", "dict-defaultdict"]},
"intermediate2": {"_type": "choice", "_value": ["normal", "inf", "neginf", "nan", "string", "dict-empty", "dict-normal", "dict-nodefault", "dict-defaultdict"]},
"intermediate3": {"_type": "choice", "_value": ["normal", "inf", "neginf", "nan", "string", "dict-empty", "dict-normal", "dict-nodefault", "dict-defaultdict"]},
"intermediate_count": {"_type": "choice", "_value": [0, 1, 2, 3]},
"final1": {"_type": "choice", "_value": ["normal", "inf", "neginf", "nan", "string", "dict-empty", "dict-normal", "dict-nodefault", "dict-defaultdict"]},
"final2": {"_type": "choice", "_value": ["normal", "inf", "neginf", "nan", "string", "dict-empty", "dict-normal", "dict-nodefault", "dict-defaultdict"]},
"final_count": {"_type": "choice", "_value": [0, 1, 2]}
},
"trainingService": {
"platform": "local"
},
"versionCheck": true,
"clusterMetaData": [
{
"key": "codeDir",
"value": "/***/nnidev/src/webui/tests/metrics-test/."
"tuner": {
"codeDirectory": "/***/nnidev/src/webui/tests/metrics-test",
"className": "naive_random.NaiveRandomTuner"
},
{
"key": "command",
"value": "python trial.py"
}
]
"trialCommand": "python trial.py",
"codeDirectory": "/***/nnidev/src/webui/tests/metrics-test"
},
"startTime": 1595901129833,
"endTime": 1595901290657
......
ts/webui/package.json
View file @ 817ec68b
......@@ -119,5 +119,8 @@
"node-forge": ">=0.10.0",
"y18n": ">=5.0.5",
"serialize-javascript": ">=5.0.1"
},
"jest": {
"verbose": true
}
}
ts/webui/src/components/Overview.tsx
View file @ 817ec68b
......@@ -60,7 +60,6 @@ class Overview extends React.Component<{}, OverviewState> {
const bestTrials = this.findBestTrials();
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const bestAccuracy = bestTrials.length > 0 ? bestTrials[0].accuracy! : NaN;
const maxExecDuration = EXPERIMENT.profile.params.maxExecDuration;
const execDuration = EXPERIMENT.profile.execDuration;
return (
......@@ -96,7 +95,7 @@ class Overview extends React.Component<{}, OverviewState> {
</TitleContext.Provider>
<ExpDurationContext.Provider
value={{
maxExecDuration,
maxExecDuration: EXPERIMENT.maxExperimentDurationSeconds,
execDuration,
updateOverviewPage,
maxDurationUnit,
......@@ -112,7 +111,7 @@ class Overview extends React.Component<{}, OverviewState> {
</TitleContext.Provider>
<ExpDurationContext.Provider
value={{
maxExecDuration,
maxExecDuration: EXPERIMENT.maxExperimentDurationSeconds,
execDuration,
updateOverviewPage,
maxDurationUnit,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment