Refactor web UI to support incremental metric loading (#1557)

* Refactor web UI to support incremental metric loading

* refactor

* Remove host job

* Move sequence ID to NNI manager

* implement incremental loading

src/nni_manager/common/experimentStartupInfo.ts

@@ -30,7 +30,6 @@ class ExperimentStartupInfo {
     private newExperiment: boolean = true;
     private basePort: number = -1;
     private initialized: boolean = false;
-    private initTrialSequenceID: number = 0;
     private logDir: string = '';
     private logLevel: string = '';
     private readonly: boolean = false;
@@ -93,17 +92,6 @@ class ExperimentStartupInfo {
 
         return this.readonly;
     }
-
-    public setInitTrialSequenceId(initSequenceId: number): void {
-        assert(this.initialized);
-        this.initTrialSequenceID = initSequenceId;
-    }
-
-    public getInitTrialSequenceId(): number {
-        assert(this.initialized);
-
-        return this.initTrialSequenceID;
-    }
 }
 
 function getExperimentId(): string {
@@ -118,14 +106,6 @@ function isNewExperiment(): boolean {
     return component.get<ExperimentStartupInfo>(ExperimentStartupInfo).isNewExperiment();
 }
 
-function setInitTrialSequenceId(initSequenceId: number): void {
-    component.get<ExperimentStartupInfo>(ExperimentStartupInfo).setInitTrialSequenceId(initSequenceId);
-}
-
-function getInitTrialSequenceId(): number {
-    return component.get<ExperimentStartupInfo>(ExperimentStartupInfo).getInitTrialSequenceId();
-}
-
 function getExperimentStartupInfo(): ExperimentStartupInfo {
     return component.get<ExperimentStartupInfo>(ExperimentStartupInfo);
 }
@@ -141,4 +121,4 @@ function isReadonly(): boolean {
 }
 
 export { ExperimentStartupInfo, getBasePort, getExperimentId, isNewExperiment, getExperimentStartupInfo,
-    setExperimentStartupInfo, setInitTrialSequenceId, getInitTrialSequenceId, isReadonly };
+    setExperimentStartupInfo, isReadonly };

src/nni_manager/common/manager.ts

@@ -83,7 +83,7 @@ interface ExperimentProfile {
     logDir?: string;
     startTime?: number;
     endTime?: number;
-    maxSequenceId: number;
+    nextSequenceId: number;
     revision: number;
 }
 
@@ -115,6 +115,9 @@ abstract class Manager {
     public abstract getClusterMetadata(key: string): Promise<string>;
 
     public abstract getMetricData(trialJobId?: string, metricType?: MetricType): Promise<MetricDataRecord[]>;
+    public abstract getMetricDataByRange(minSeqId: number, maxSeqId: number): Promise<MetricDataRecord[]>;
+    public abstract getLatestMetricData(): Promise<MetricDataRecord[]>;
+
     public abstract getTrialJobStatistics(): Promise<TrialJobStatistics[]>;
     public abstract getStatus(): NNIManagerStatus;
 }

src/nni_manager/common/trainingService.ts

@@ -23,20 +23,12 @@
  * define TrialJobStatus
  */
 type TrialJobStatus = 'UNKNOWN' | 'WAITING' | 'RUNNING' | 'SUCCEEDED' | 'FAILED' | 'USER_CANCELED' | 'SYS_CANCELED' | 'EARLY_STOPPED';
-type JobType = 'TRIAL' | 'HOST';
 
 interface TrainingServiceMetadata {
     readonly key: string;
     readonly value: string;
 }
 
-/**
- * define JobApplicationForm
- */
-interface JobApplicationForm {
-    readonly jobType: JobType;
-}
-
 interface HyperParameters {
     readonly value: string;
     readonly index: number;
@@ -45,18 +37,11 @@ interface HyperParameters {
 /**
  * define TrialJobApplicationForm
  */
-interface TrialJobApplicationForm extends JobApplicationForm {
+interface TrialJobApplicationForm {
+    readonly sequenceId: number;
     readonly hyperParameters: HyperParameters;
 }
 
-/**
- * define HostJobApplicationForm
- */
-interface HostJobApplicationForm extends JobApplicationForm {
-    readonly host: string;
-    readonly cmd: string;
-}
-
 /**
  * define TrialJobDetail
  */
@@ -69,8 +54,7 @@ interface TrialJobDetail {
     readonly tags?: string[];
     readonly url?: string;
     readonly workingDirectory: string;
-    readonly form: JobApplicationForm;
-    readonly sequenceId: number;
+    readonly form: TrialJobApplicationForm;
     isEarlyStopped?: boolean;
 }
 
@@ -112,8 +96,8 @@ abstract class TrainingService {
     public abstract getTrialJob(trialJobId: string): Promise<TrialJobDetail>;
     public abstract addTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void;
     public abstract removeTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void;
-    public abstract submitTrialJob(form: JobApplicationForm): Promise<TrialJobDetail>;
-    public abstract updateTrialJob(trialJobId: string, form: JobApplicationForm): Promise<TrialJobDetail>;
+    public abstract submitTrialJob(form: TrialJobApplicationForm): Promise<TrialJobDetail>;
+    public abstract updateTrialJob(trialJobId: string, form: TrialJobApplicationForm): Promise<TrialJobDetail>;
     public abstract get isMultiPhaseJobSupported(): boolean;
     public abstract cancelTrialJob(trialJobId: string, isEarlyStopped?: boolean): Promise<void>;
     public abstract setClusterMetadata(key: string, value: string): Promise<void>;
@@ -135,5 +119,5 @@ class NNIManagerIpConfig {
 export {
     TrainingService, TrainingServiceError, TrialJobStatus, TrialJobApplicationForm,
     TrainingServiceMetadata, TrialJobDetail, TrialJobMetric, HyperParameters,
-    HostJobApplicationForm, JobApplicationForm, JobType, NNIManagerIpConfig
+    NNIManagerIpConfig
 };

src/nni_manager/core/nnimanager.ts

@@ -26,7 +26,7 @@ import { Deferred } from 'ts-deferred';
 import * as component from '../common/component';
 import { DataStore, MetricDataRecord, MetricType, TrialJobInfo } from '../common/datastore';
 import { NNIError } from '../common/errors';
-import { getExperimentId, setInitTrialSequenceId } from '../common/experimentStartupInfo';
+import { getExperimentId } from '../common/experimentStartupInfo';
 import { getLogger, Logger } from '../common/log';
 import {
     ExperimentParams, ExperimentProfile, Manager, ExperimentStatus,
@@ -62,7 +62,7 @@ class NNIManager implements Manager {
     private readonly: boolean;
 
     private trialJobMetricListener: (metric: TrialJobMetric) => void;
-    
+
     constructor() {
         this.currSubmittedTrialNum = 0;
         this.trialConcurrencyChange = 0;
@@ -204,7 +204,6 @@ class NNIManager implements Manager {
             return Promise.resolve();
         }
         const expParams: ExperimentParams = this.experimentProfile.params;
-        setInitTrialSequenceId(this.experimentProfile.maxSequenceId + 1);
 
         // Set up multiphase config
         if (expParams.multiPhase && this.trainingService.isMultiPhaseJobSupported) {
@@ -301,6 +300,37 @@ class NNIManager implements Manager {
         return this.dataStore.getMetricData(trialJobId, metricType);
     }
 
+    public async getMetricDataByRange(minSeqId: number, maxSeqId: number): Promise<MetricDataRecord[]> {
+        const trialJobs = await this.dataStore.listTrialJobs();
+        const targetTrials = trialJobs.filter(trial => (
+            // FIXME: can this be undefined?
+            trial.sequenceId !== undefined && minSeqId <= trial.sequenceId && trial.sequenceId <= maxSeqId
+        ));
+        const targetTrialIds = new Set(targetTrials.map(trial => trial.id));
+
+        const allMetrics = await this.dataStore.getMetricData();
+        return allMetrics.filter(metric => targetTrialIds.has(metric.trialJobId));
+    }
+
+    public async getLatestMetricData(): Promise<MetricDataRecord[]> {
+        // FIXME: this can take a long time
+        const allMetrics: MetricDataRecord[] = await this.dataStore.getMetricData();
+        const finals: MetricDataRecord[] = [];
+        const latestIntermediates: Map<string, MetricDataRecord> = new Map<string, MetricDataRecord>();
+        for (const metric of allMetrics) {
+            if (metric.type !== 'PERIODICAL') {
+                finals.push(metric);
+            } else {
+                const old: MetricDataRecord | undefined = latestIntermediates.get(metric.trialJobId);
+                if (old === undefined || old.sequence <= metric.sequence) {
+                    latestIntermediates.set(metric.trialJobId, metric);
+                }
+            }
+        }
+        return finals.concat(Array.from(latestIntermediates.values()));
+        // FIXME: unit test
+    }
+
     public getExperimentProfile(): Promise<ExperimentProfile> {
         // TO DO: using Promise.resolve()
         const deferred: Deferred<ExperimentProfile> = new Deferred<ExperimentProfile>();
@@ -383,7 +413,7 @@ class NNIManager implements Manager {
         if (this.dispatcher === undefined) {
             throw new Error('Error: tuner has not been setup');
         }
-        this.trainingService.removeTrialJobMetricListener(this.trialJobMetricListener); 
+        this.trainingService.removeTrialJobMetricListener(this.trialJobMetricListener);
         this.dispatcher.sendCommand(TERMINATE);
         let tunerAlive: boolean = true;
         // gracefully terminate tuner and assessor here, wait at most 30 seconds.
@@ -456,11 +486,7 @@ class NNIManager implements Manager {
                 case 'EARLY_STOPPED':
                     this.trialJobs.delete(trialJobId);
                     finishedTrialJobNum++;
-                    if (trialJobDetail.form.jobType === 'TRIAL') {
-                        hyperParams = (<TrialJobApplicationForm>trialJobDetail.form).hyperParameters.value;
-                    } else {
-                        throw new Error('Error: jobType error, not TRIAL');
-                    }
+                    hyperParams = trialJobDetail.form.hyperParameters.value;
                     this.dispatcher.sendCommand(TRIAL_END, JSON.stringify({
                         trial_job_id: trialJobDetail.id,
                         event: trialJobDetail.status,
@@ -473,11 +499,7 @@ class NNIManager implements Manager {
                     // TO DO: push this job to queue for retry
                     this.trialJobs.delete(trialJobId);
                     finishedTrialJobNum++;
-                    if (trialJobDetail.form.jobType === 'TRIAL') {
-                        hyperParams = (<TrialJobApplicationForm>trialJobDetail.form).hyperParameters.value;
-                    } else {
-                        throw new Error('Error: jobType error, not TRIAL');
-                    }
+                    hyperParams = trialJobDetail.form.hyperParameters.value;
                     this.dispatcher.sendCommand(TRIAL_END, JSON.stringify({
                         trial_job_id: trialJobDetail.id,
                         event: trialJobDetail.status,
@@ -576,7 +598,7 @@ class NNIManager implements Manager {
                     }
                     this.currSubmittedTrialNum++;
                     const trialJobAppForm: TrialJobApplicationForm = {
-                        jobType: 'TRIAL',
+                        sequenceId: this.experimentProfile.nextSequenceId++,
                         hyperParameters: {
                             value: hyperParams,
                             index: 0
@@ -584,7 +606,7 @@ class NNIManager implements Manager {
                     };
                     this.log.info(`submitTrialJob: form: ${JSON.stringify(trialJobAppForm)}`);
                     const trialJobDetail: TrialJobDetail = await this.trainingService.submitTrialJob(trialJobAppForm);
-                    await this.storeMaxSequenceId(trialJobDetail.sequenceId);
+                    await this.storeExperimentProfile();
                     this.trialJobs.set(trialJobDetail.id, Object.assign({}, trialJobDetail));
                     const trialJobDetailSnapshot: TrialJobDetail | undefined = this.trialJobs.get(trialJobDetail.id);
                     if (trialJobDetailSnapshot != undefined) {
@@ -703,7 +725,7 @@ class NNIManager implements Manager {
                 assert(tunerCommand.trial_job_id !== undefined);
 
                 const trialJobForm: TrialJobApplicationForm = {
-                    jobType: 'TRIAL',
+                    sequenceId: -1,  // FIXME: multi-phase tuner should use sequence ID instead of trial job ID
                     hyperParameters: {
                         value: content,
                         index: tunerCommand.parameter_index
@@ -757,7 +779,7 @@ class NNIManager implements Manager {
             revision: 0,
             execDuration: 0,
             logDir: getExperimentRootDir(),
-            maxSequenceId: 0,
+            nextSequenceId: 0,
             params: {
                 authorName: '',
                 experimentName: '',
@@ -788,13 +810,6 @@ class NNIManager implements Manager {
 
         return Promise.resolve(chkpDir);
     }
-
-    private async storeMaxSequenceId(sequenceId: number): Promise<void> {
-        if (sequenceId > this.experimentProfile.maxSequenceId) {
-            this.experimentProfile.maxSequenceId = sequenceId;
-            await this.storeExperimentProfile();
-        }
-    }
 }
 
 export { NNIManager };

src/nni_manager/core/sqlDatabase.ts

@@ -54,7 +54,7 @@ create table ExperimentProfile (
     startTime integer,
     endTime integer,
     logDir text,
-    maxSequenceId integer,
+    nextSequenceId integer,
     revision integer);
 create index ExperimentProfile_id on ExperimentProfile(id);
 `;
@@ -67,7 +67,7 @@ function loadExperimentProfile(row: any): ExperimentProfile {
         startTime: row.startTime === null ? undefined : row.startTime,
         endTime: row.endTime === null ? undefined : row.endTime,
         logDir: row.logDir === null ? undefined : row.logDir,
-        maxSequenceId: row.maxSequenceId,
+        nextSequenceId: row.nextSequenceId,
         revision: row.revision
     };
 }
@@ -144,7 +144,7 @@ class SqlDB implements Database {
             exp.startTime === undefined ? null : exp.startTime,
             exp.endTime === undefined ? null : exp.endTime,
             exp.logDir === undefined ? null : exp.logDir,
-            exp.maxSequenceId,
+            exp.nextSequenceId,
             exp.revision
         ];
         this.log.trace(`storeExperimentProfile: SQL: ${sql}, args: ${JSON.stringify(args)}`);
@@ -183,7 +183,7 @@ class SqlDB implements Database {
         event: TrialJobEvent, trialJobId: string, timestamp: number, hyperParameter?: string, jobDetail?: TrialJobDetail): Promise<void> {
         const sql: string = 'insert into TrialJobEvent values (?,?,?,?,?,?)';
         const logPath: string | undefined = jobDetail === undefined ? undefined : jobDetail.url;
-        const sequenceId: number | undefined = jobDetail === undefined ? undefined : jobDetail.sequenceId;
+        const sequenceId: number | undefined = jobDetail === undefined ? undefined : jobDetail.form.sequenceId;
         const args: any[] = [timestamp, trialJobId, event, hyperParameter, logPath, sequenceId];
 
         this.log.trace(`storeTrialJobEvent: SQL: ${sql}, args: ${JSON.stringify(args)}`);

src/nni_manager/core/test/dataStore.test.ts

@@ -80,7 +80,7 @@ describe('Unit test for dataStore', () => {
             execDuration: 0,
             startTime: Date.now(),
             endTime: Date.now(),
-            maxSequenceId: 0,
+            nextSequenceId: 0,
             revision: 0
         }
         const id: string = profile.id;

src/nni_manager/core/test/mockedTrainingService.ts

@@ -41,9 +41,9 @@ class MockedTrainingService extends TrainingService {
         url: 'http://test',
         workingDirectory: '/tmp/mocked',
         form: {
-            jobType: 'TRIAL'
+            sequenceId: 0,
+            hyperParameters: { value: '', index: 0 }
         },
-        sequenceId: 0
     };
     public jobDetail2: TrialJobDetail = {
         id: '3456',
@@ -55,9 +55,9 @@ class MockedTrainingService extends TrainingService {
         url: 'http://test',
         workingDirectory: '/tmp/mocked',
         form: {
-            jobType: 'TRIAL'
+            sequenceId: 1,
+            hyperParameters: { value: '', index: 1 }
         },
-        sequenceId: 0
     };
 
     public listTrialJobs(): Promise<TrialJobDetail[]> {

src/nni_manager/core/test/nnimanager.test.ts

@@ -101,7 +101,7 @@ describe('Unit test for nnimanager', function () {
         params: updateExperimentParams,
         id: 'test',
         execDuration: 0,
-        maxSequenceId: 0,
+        nextSequenceId: 0,
         revision: 0
     }
 

src/nni_manager/core/test/sqlDatabase.test.ts

@@ -64,10 +64,10 @@ const expParams2: ExperimentParams = {
 };
 
 const profiles: ExperimentProfile[] = [
-    { params: expParams1, id: '#1', execDuration: 0, logDir: '/log', startTime: Date.now(), endTime: undefined, maxSequenceId: 0, revision: 1,},
-    { params: expParams1, id: '#1', execDuration: 0, logDir: '/log', startTime: Date.now(), endTime: Date.now(), maxSequenceId: 0, revision: 2 },
-    { params: expParams2, id: '#2', execDuration: 0, logDir: '/log', startTime: Date.now(), endTime: Date.now(), maxSequenceId: 0, revision: 2 },
-    { params: expParams2, id: '#2', execDuration: 0, logDir: '/log', startTime: Date.now(), endTime: Date.now(), maxSequenceId: 0, revision: 3 }
+    { params: expParams1, id: '#1', execDuration: 0, logDir: '/log', startTime: Date.now(), endTime: undefined, nextSequenceId: 0, revision: 1,},
+    { params: expParams1, id: '#1', execDuration: 0, logDir: '/log', startTime: Date.now(), endTime: Date.now(), nextSequenceId: 1, revision: 2 },
+    { params: expParams2, id: '#2', execDuration: 0, logDir: '/log', startTime: Date.now(), endTime: Date.now(), nextSequenceId: 0, revision: 2 },
+    { params: expParams2, id: '#2', execDuration: 0, logDir: '/log', startTime: Date.now(), endTime: Date.now(), nextSequenceId: 2, revision: 3 }
 ];
 
 const events: TrialJobEventRecord[] = [

src/nni_manager/rest_server/restHandler.ts

@@ -72,6 +72,8 @@ class NNIRestHandler {
         this.addTrialJob(router);
         this.cancelTrialJob(router);
         this.getMetricData(router);
+        this.getMetricDataByRange(router);
+        this.getLatestMetricData(router);
         this.exportData(router);
 
         // Express-joi-validator configuration
@@ -262,6 +264,28 @@ class NNIRestHandler {
         });
     }
 
+    private getMetricDataByRange(router: Router): void {
+        router.get('/metric-data-range/:min_seq_id/:max_seq_id', async (req: Request, res: Response) => {
+            const minSeqId = Number(req.params.min_seq_id);
+            const maxSeqId = Number(req.params.max_seq_id);
+            this.nniManager.getMetricDataByRange(minSeqId, maxSeqId).then((metricsData: MetricDataRecord[]) => {
+                res.send(metricsData);
+            }).catch((err: Error) => {
+                this.handle_error(err, res);
+            });
+        });
+    }
+
+    private getLatestMetricData(router: Router): void {
+        router.get('/metric-data-latest/', async (req: Request, res: Response) => {
+            this.nniManager.getLatestMetricData().then((metricsData: MetricDataRecord[]) => {
+                res.send(metricsData);
+            }).catch((err: Error) => {
+                this.handle_error(err, res);
+            });
+        });
+    }
+
     private exportData(router: Router): void {
         router.get('/export-data', (req: Request, res: Response) => {
             this.nniManager.exportData().then((exportedData: string) => {

src/nni_manager/rest_server/restValidationSchemas.ts

@@ -209,7 +209,7 @@ export namespace ValidationSchemas {
             startTime: joi.number(),
             endTime: joi.number(),
             logDir: joi.string(),
-            maxSequenceId: joi.number()
+            nextSequenceId: joi.number()
         }
     };
 }

src/nni_manager/rest_server/test/mockedNNIManager.ts

@@ -85,9 +85,9 @@ export class MockedNNIManager extends Manager {
             // tslint:disable-next-line:no-http-string
             url: 'http://test',
             workingDirectory: '/tmp/mocked',
-            sequenceId: 0,
             form: {
-                jobType: 'TRIAL'
+                sequenceId: 0,
+                hyperParameters: { value: '', index: 0 }
             }
         };
         deferred.resolve(jobDetail);
@@ -129,6 +129,12 @@ export class MockedNNIManager extends Manager {
     public getMetricData(trialJobId: string, metricType: MetricType): Promise<MetricDataRecord[]> {
         throw new MethodNotImplementedError();
     }
+    public getMetricDataByRange(minSeqId: number, maxSeqId: number): Promise<MetricDataRecord[]> {
+        throw new MethodNotImplementedError();
+    }
+    public getLatestMetricData(): Promise<MetricDataRecord[]> {
+        throw new MethodNotImplementedError();
+    }
     public getExperimentProfile(): Promise<ExperimentProfile> {
         const profile: ExperimentProfile = {
             params: {
@@ -148,7 +154,7 @@ export class MockedNNIManager extends Manager {
             execDuration: 0,
             startTime: Date.now(),
             endTime: Date.now(),
-            maxSequenceId: 0,
+            nextSequenceId: 0,
             revision: 0
         };
 

src/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts

@@ -25,7 +25,7 @@ import * as path from 'path';
 import * as component from '../../../common/component';
 import { getExperimentId } from '../../../common/experimentStartupInfo';
 import {
-    JobApplicationForm, NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus
+    NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus
 } from '../../../common/trainingService';
 import { delay, generateParamFileName, getExperimentRootDir, uniqueString } from '../../../common/utils';
 import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../../common/containerJobData';
@@ -55,7 +55,6 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple
         super();
         this.fcJobInfoCollector = new FrameworkControllerJobInfoCollector(this.trialJobsMap);
         this.experimentId = getExperimentId();
-        this.nextTrialSequenceId = -1;
     }
 
     public async run(): Promise<void> {
@@ -77,7 +76,7 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple
         }
     }
 
-    public async submitTrialJob(form: JobApplicationForm): Promise<TrialJobDetail> {
+    public async submitTrialJob(form: TrialJobApplicationForm): Promise<TrialJobDetail> {
         if (this.fcClusterConfig === undefined) {
             throw new Error('frameworkcontrollerClusterConfig is not initialized');
         }
@@ -91,14 +90,13 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple
         }
 
         const trialJobId: string = uniqueString(5);
-        const curTrialSequenceId: number = this.generateSequenceId();
         // Set trial's NFS working folder
         const trialWorkingFolder: string = path.join(this.CONTAINER_MOUNT_PATH, 'nni', getExperimentId(), trialJobId);
         const trialLocalTempFolder: string = path.join(getExperimentRootDir(), 'trials-local', trialJobId);
         const frameworkcontrollerJobName: string = `nniexp${this.experimentId}trial${trialJobId}`.toLowerCase();
         //Generate the port used for taskRole
         this.generateContainerPort();
-        await this.prepareRunScript(trialLocalTempFolder, curTrialSequenceId, trialJobId, trialWorkingFolder, form);
+        await this.prepareRunScript(trialLocalTempFolder, trialJobId, trialWorkingFolder, form);
 
         //upload code files
         const trialJobOutputUrl: string = await this.uploadCodeFiles(trialJobId, trialLocalTempFolder);
@@ -113,7 +111,6 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple
             trialWorkingFolder,
             form,
             frameworkcontrollerJobName,
-            curTrialSequenceId,
             trialJobOutputUrl
         );
 
@@ -248,8 +245,8 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple
         return `${portScript} . /mnt/frameworkbarrier/injector.sh && ${command}`;
     }
 
-    private async prepareRunScript(trialLocalTempFolder: string, curTrialSequenceId: number, trialJobId: string,
-                                   trialWorkingFolder: string, form: JobApplicationForm): Promise<void> {
+    private async prepareRunScript(trialLocalTempFolder: string, trialJobId: string,
+                                   trialWorkingFolder: string, form: TrialJobApplicationForm): Promise<void> {
         if (this.fcTrialConfig === undefined) {
             throw new Error('frameworkcontroller trial config is not initialized');
         }
@@ -264,16 +261,16 @@ class FrameworkControllerTrainingService extends KubernetesTrainingService imple
         for (const taskRole of this.fcTrialConfig.taskRoles) {
             const runScriptContent: string =
               await this.generateRunScript('frameworkcontroller', trialJobId, trialWorkingFolder,
-                                           this.generateCommandScript(taskRole.command), curTrialSequenceId.toString(),
+                                           this.generateCommandScript(taskRole.command), form.sequenceId.toString(),
                                            taskRole.name, taskRole.gpuNum);
             await fs.promises.writeFile(path.join(trialLocalTempFolder, `run_${taskRole.name}.sh`), runScriptContent, { encoding: 'utf8' });
         }
 
         // Write file content ( parameter.cfg ) to local tmp folders
         const trialForm : TrialJobApplicationForm = (<TrialJobApplicationForm>form);
-        if (trialForm !== undefined && trialForm.hyperParameters !== undefined) {
-            await fs.promises.writeFile(path.join(trialLocalTempFolder, generateParamFileName(trialForm.hyperParameters)),
-                                        trialForm.hyperParameters.value, { encoding: 'utf8' });
+        if (form !== undefined) {
+            await fs.promises.writeFile(path.join(trialLocalTempFolder, generateParamFileName(form.hyperParameters)),
+                                        form.hyperParameters.value, { encoding: 'utf8' });
         }
     }
 

src/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts

@@ -27,7 +27,7 @@ import * as component from '../../../common/component';
 
 import { getExperimentId } from '../../../common/experimentStartupInfo';
 import {
-    JobApplicationForm, NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus
+    NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus
 } from '../../../common/trainingService';
 import { delay, generateParamFileName, getExperimentRootDir, uniqueString } from '../../../common/utils';
 import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../../common/containerJobData';
@@ -59,7 +59,6 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber
         super();
         this.kubeflowJobInfoCollector = new KubeflowJobInfoCollector(this.trialJobsMap);
         this.experimentId = getExperimentId();
-        this.nextTrialSequenceId = -1;
         this.log.info('Construct Kubeflow training service.');
     }
 
@@ -84,7 +83,7 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber
         this.log.info('Kubeflow training service exit.');
     }
 
-    public async submitTrialJob(form: JobApplicationForm): Promise<TrialJobDetail> {
+    public async submitTrialJob(form: TrialJobApplicationForm): Promise<TrialJobDetail> {
         if (this.kubernetesCRDClient === undefined) {
             throw new Error('Kubeflow job operator client is undefined');
         }
@@ -96,10 +95,9 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber
         const trialJobId: string = uniqueString(5);
         const trialWorkingFolder: string = path.join(this.CONTAINER_MOUNT_PATH, 'nni', getExperimentId(), trialJobId);
         const kubeflowJobName: string = `nni-exp-${this.experimentId}-trial-${trialJobId}`.toLowerCase();
-        const curTrialSequenceId: number = this.generateSequenceId();
         const trialLocalTempFolder: string = path.join(getExperimentRootDir(), 'trials-local', trialJobId);
         //prepare the runscript
-        await this.prepareRunScript(trialLocalTempFolder, trialJobId, trialWorkingFolder, curTrialSequenceId, form);
+        await this.prepareRunScript(trialLocalTempFolder, trialJobId, trialWorkingFolder, form);
         //upload files to sotrage
         const trialJobOutputUrl: string = await this.uploadCodeFiles(trialJobId, trialLocalTempFolder);
         let initStatus: TrialJobStatus = 'WAITING';
@@ -113,7 +111,6 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber
             trialWorkingFolder,
             form,
             kubeflowJobName,
-            curTrialSequenceId,
             trialJobOutputUrl
         );
 
@@ -236,8 +233,8 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber
         return Promise.resolve(trialJobOutputUrl);
     }
 
-    private async prepareRunScript(trialLocalTempFolder: string, trialJobId: string, trialWorkingFolder: string, curTrialSequenceId: number,
-                                   form: JobApplicationForm): Promise<void> {
+    private async prepareRunScript(trialLocalTempFolder: string, trialJobId: string, trialWorkingFolder: string,
+                                   form: TrialJobApplicationForm): Promise<void> {
         if (this.kubeflowClusterConfig === undefined) {
             throw new Error('Kubeflow Cluster config is not initialized');
         }
@@ -262,7 +259,7 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber
         if (kubeflowTrialConfig.worker !== undefined) {
            const workerRunScriptContent: string = await this.generateRunScript('kubeflow', trialJobId, trialWorkingFolder,
                                                                                kubeflowTrialConfig.worker.command,
-                                                                               curTrialSequenceId.toString(), 'worker',
+                                                                               form.sequenceId.toString(), 'worker',
                                                                                kubeflowTrialConfig.worker.gpuNum);
            await fs.promises.writeFile(path.join(trialLocalTempFolder, 'run_worker.sh'), workerRunScriptContent, { encoding: 'utf8' });
         }
@@ -272,7 +269,7 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber
            if (tensorflowTrialConfig.ps !== undefined) {
                const psRunScriptContent: string = await this.generateRunScript('kubeflow', trialJobId, trialWorkingFolder,
                                                                                tensorflowTrialConfig.ps.command,
-                                                                               curTrialSequenceId.toString(),
+                                                                               form.sequenceId.toString(),
                                                                                'ps', tensorflowTrialConfig.ps.gpuNum);
                await fs.promises.writeFile(path.join(trialLocalTempFolder, 'run_ps.sh'), psRunScriptContent, { encoding: 'utf8' });
            }
@@ -281,16 +278,15 @@ class KubeflowTrainingService extends KubernetesTrainingService implements Kuber
            if (pytorchTrialConfig.master !== undefined) {
                const masterRunScriptContent: string = await this.generateRunScript('kubeflow', trialJobId, trialWorkingFolder,
                                                                                    pytorchTrialConfig.master.command,
-                                                                                   curTrialSequenceId.toString(), 'master',
+                                                                                   form.sequenceId.toString(), 'master',
                                                                                    pytorchTrialConfig.master.gpuNum);
                await fs.promises.writeFile(path.join(trialLocalTempFolder, 'run_master.sh'), masterRunScriptContent, { encoding: 'utf8' });
            }
         }
         // Write file content ( parameter.cfg ) to local tmp folders
-        const trialForm : TrialJobApplicationForm = (<TrialJobApplicationForm>form);
-        if (trialForm !== undefined && trialForm.hyperParameters !== undefined) {
-           await fs.promises.writeFile(path.join(trialLocalTempFolder, generateParamFileName(trialForm.hyperParameters)),
-                                       trialForm.hyperParameters.value, { encoding: 'utf8' });
+        if (form !== undefined) {
+           await fs.promises.writeFile(path.join(trialLocalTempFolder, generateParamFileName(form.hyperParameters)),
+                                       form.hyperParameters.value, { encoding: 'utf8' });
         }
     }
 

src/nni_manager/training_service/kubernetes/kubernetesData.ts

@@ -19,7 +19,7 @@
 
 'use strict';
 
-import { JobApplicationForm, TrialJobDetail, TrialJobStatus  } from '../../common/trainingService';
+import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus  } from '../../common/trainingService';
 
 /**
  * KubeflowTrialJobDetail
@@ -33,21 +33,19 @@ export class KubernetesTrialJobDetail implements TrialJobDetail {
     public tags?: string[];
     public url?: string;
     public workingDirectory: string;
-    public form: JobApplicationForm;
+    public form: TrialJobApplicationForm;
     public kubernetesJobName: string;
-    public sequenceId: number;
     public queryJobFailedCount: number;
 
     constructor(id: string, status: TrialJobStatus, submitTime: number,
-                workingDirectory: string, form: JobApplicationForm,
-                kubernetesJobName: string, sequenceId: number, url: string) {
+                workingDirectory: string, form: TrialJobApplicationForm,
+                kubernetesJobName: string, url: string) {
         this.id = id;
         this.status = status;
         this.submitTime = submitTime;
         this.workingDirectory = workingDirectory;
         this.form = form;
         this.kubernetesJobName = kubernetesJobName;
-        this.sequenceId = sequenceId;
         this.tags = [];
         this.queryJobFailedCount = 0;
         this.url = url;

src/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts

@@ -26,7 +26,7 @@ import * as azureStorage from 'azure-storage';
 import { EventEmitter } from 'events';
 import { Base64 } from 'js-base64';
 import { String } from 'typescript-string-operations';
-import { getExperimentId, getInitTrialSequenceId } from '../../common/experimentStartupInfo';
+import { getExperimentId } from '../../common/experimentStartupInfo';
 import { getLogger, Logger } from '../../common/log';
 import {
     NNIManagerIpConfig, TrialJobDetail, TrialJobMetric
@@ -53,7 +53,6 @@ abstract class KubernetesTrainingService {
     protected readonly trialLocalNFSTempFolder: string;
     protected stopping: boolean = false;
     protected experimentId! : string;
-    protected nextTrialSequenceId: number;
     protected kubernetesRestServerPort?: number;
     protected readonly CONTAINER_MOUNT_PATH: string;
     protected azureStorageClient?: azureStorage.FileService;
@@ -74,7 +73,6 @@ abstract class KubernetesTrainingService {
         this.trialJobsMap = new Map<string, KubernetesTrialJobDetail>();
         this.trialLocalNFSTempFolder = path.join(getExperimentRootDir(), 'trials-nfs-tmp');
         this.experimentId = getExperimentId();
-        this.nextTrialSequenceId = -1;
         this.CONTAINER_MOUNT_PATH = '/tmp/mount';
         this.genericK8sClient = new GeneralK8sClient();
         this.logCollection = 'none';
@@ -93,9 +91,7 @@ abstract class KubernetesTrainingService {
         const jobs: TrialJobDetail[] = [];
 
         for (const [key, value] of this.trialJobsMap) {
-            if (value.form.jobType === 'TRIAL') {
-                jobs.push(await this.getTrialJob(key));
-            }
+            jobs.push(await this.getTrialJob(key));
         }
 
         return Promise.resolve(jobs);
@@ -222,14 +218,6 @@ abstract class KubernetesTrainingService {
         return Promise.resolve();
     }
 
-    protected generateSequenceId(): number {
-        if (this.nextTrialSequenceId === -1) {
-            this.nextTrialSequenceId = getInitTrialSequenceId();
-        }
-
-        return this.nextTrialSequenceId++;
-    }
-
     // tslint:disable: no-unsafe-any no-any
     protected async createAzureStorage(vaultName: string, valutKeyName: string, accountName: string, azureShare: string): Promise<void> {
         try {

src/nni_manager/training_service/local/localTrainingService.ts

@@ -26,10 +26,10 @@ import * as path from 'path';
 import * as ts from 'tail-stream';
 import * as tkill from 'tree-kill';
 import { NNIError, NNIErrorNames } from '../../common/errors';
-import { getExperimentId, getInitTrialSequenceId } from '../../common/experimentStartupInfo';
+import { getExperimentId } from '../../common/experimentStartupInfo';
 import { getLogger, Logger } from '../../common/log';
 import {
-    HostJobApplicationForm, HyperParameters, JobApplicationForm, TrainingService, TrialJobApplicationForm,
+    HyperParameters, TrainingService, TrialJobApplicationForm,
     TrialJobDetail, TrialJobMetric, TrialJobStatus
 } from '../../common/trainingService';
 import {
@@ -76,21 +76,19 @@ class LocalTrialJobDetail implements TrialJobDetail {
     public tags?: string[];
     public url?: string;
     public workingDirectory: string;
-    public form: JobApplicationForm;
-    public sequenceId: number;
+    public form: TrialJobApplicationForm;
     public pid?: number;
     public gpuIndices?: number[];
 
     constructor(
         id: string, status: TrialJobStatus, submitTime: number,
-        workingDirectory: string, form: JobApplicationForm, sequenceId: number) {
+        workingDirectory: string, form: TrialJobApplicationForm) {
         this.id = id;
         this.status = status;
         this.submitTime = submitTime;
         this.workingDirectory = workingDirectory;
         this.form = form;
         this.url = `file://localhost:${workingDirectory}`;
-        this.sequenceId = sequenceId;
         this.gpuIndices = [];
     }
 }
@@ -125,7 +123,6 @@ class LocalTrainingService implements TrainingService {
     private initialized: boolean;
     private stopping: boolean;
     private rootDir!: string;
-    private trialSequenceId: number;
     private readonly experimentId! : string;
     private gpuScheduler!: GPUScheduler;
     private readonly occupiedGpuIndexNumMap: Map<number, number>;
@@ -145,7 +142,6 @@ class LocalTrainingService implements TrainingService {
         this.initialized = false;
         this.stopping = false;
         this.log = getLogger();
-        this.trialSequenceId = -1;
         this.experimentId = getExperimentId();
         this.jobStreamMap = new Map<string, ts.Stream>();
         this.log.info('Construct local machine training service.');
@@ -169,9 +165,7 @@ class LocalTrainingService implements TrainingService {
         const jobs: TrialJobDetail[] = [];
         for (const key of this.jobMap.keys()) {
             const trialJob: TrialJobDetail = await this.getTrialJob(key);
-            if (trialJob.form.jobType === 'TRIAL') {
-                jobs.push(trialJob);
-            }
+            jobs.push(trialJob);
         }
 
         return jobs;
@@ -182,9 +176,6 @@ class LocalTrainingService implements TrainingService {
         if (trialJob === undefined) {
             throw new NNIError(NNIErrorNames.NOT_FOUND, 'Trial job not found');
         }
-        if (trialJob.form.jobType === 'HOST') {
-            return this.getHostJob(trialJobId);
-        }
         if (trialJob.status === 'RUNNING') {
             const alive: boolean = await isAlive(trialJob.pid);
             if (!alive) {
@@ -219,28 +210,21 @@ class LocalTrainingService implements TrainingService {
         this.eventEmitter.off('metric', listener);
     }
 
-    public submitTrialJob(form: JobApplicationForm): Promise<TrialJobDetail> {
-        if (form.jobType === 'HOST') {
-            return this.runHostJob(<HostJobApplicationForm>form);
-        } else if (form.jobType === 'TRIAL') {
-            const trialJobId: string = uniqueString(5);
-            const trialJobDetail: LocalTrialJobDetail = new LocalTrialJobDetail(
-                trialJobId,
-                'WAITING',
-                Date.now(),
-                path.join(this.rootDir, 'trials', trialJobId),
-                form,
-                this.generateSequenceId()
-            );
-            this.jobQueue.push(trialJobId);
-            this.jobMap.set(trialJobId, trialJobDetail);
-
-            this.log.debug(`submitTrialJob: return: ${JSON.stringify(trialJobDetail)} `);
-
-            return Promise.resolve(trialJobDetail);
-        } else {
-            return Promise.reject(new Error(`Job form not supported: ${JSON.stringify(form)}`));
-        }
+    public submitTrialJob(form: TrialJobApplicationForm): Promise<TrialJobDetail> {
+        const trialJobId: string = uniqueString(5);
+        const trialJobDetail: LocalTrialJobDetail = new LocalTrialJobDetail(
+            trialJobId,
+            'WAITING',
+            Date.now(),
+            path.join(this.rootDir, 'trials', trialJobId),
+            form
+        );
+        this.jobQueue.push(trialJobId);
+        this.jobMap.set(trialJobId, trialJobDetail);
+
+        this.log.debug(`submitTrialJob: return: ${JSON.stringify(trialJobDetail)} `);
+
+        return Promise.resolve(trialJobDetail);
     }
 
     /**
@@ -248,16 +232,12 @@ class LocalTrainingService implements TrainingService {
      * @param trialJobId trial job id
      * @param form job application form
      */
-    public async updateTrialJob(trialJobId: string, form: JobApplicationForm): Promise<TrialJobDetail> {
+    public async updateTrialJob(trialJobId: string, form: TrialJobApplicationForm): Promise<TrialJobDetail> {
         const trialJobDetail: undefined | TrialJobDetail = this.jobMap.get(trialJobId);
         if (trialJobDetail === undefined) {
             throw new Error(`updateTrialJob failed: ${trialJobId} not found`);
         }
-        if (form.jobType === 'TRIAL') {
-            await this.writeParameterFile(trialJobDetail.workingDirectory, (<TrialJobApplicationForm>form).hyperParameters);
-        } else {
-            throw new Error(`updateTrialJob failed: jobType ${form.jobType} not supported.`);
-        }
+        await this.writeParameterFile(trialJobDetail.workingDirectory, form.hyperParameters);
 
         return trialJobDetail;
     }
@@ -279,13 +259,7 @@ class LocalTrainingService implements TrainingService {
 
             return Promise.resolve();
         }
-        if (trialJob.form.jobType === 'TRIAL') {
-            tkill(trialJob.pid, 'SIGKILL');
-        } else if (trialJob.form.jobType === 'HOST') {
-            await cpp.exec(`pkill -9 -P ${trialJob.pid}`);
-        } else {
-            throw new Error(`Job type not supported: ${trialJob.form.jobType}`);
-        }
+        tkill(trialJob.pid, 'SIGKILL');
         this.setTrialJobStatus(trialJob, getJobCancelStatus(isEarlyStopped));
 
         return Promise.resolve();
@@ -409,7 +383,7 @@ class LocalTrainingService implements TrainingService {
             { key: 'NNI_SYS_DIR', value: trialJobDetail.workingDirectory },
             { key: 'NNI_TRIAL_JOB_ID', value: trialJobDetail.id },
             { key: 'NNI_OUTPUT_DIR', value: trialJobDetail.workingDirectory },
-            { key: 'NNI_TRIAL_SEQ_ID', value: trialJobDetail.sequenceId.toString() },
+            { key: 'NNI_TRIAL_SEQ_ID', value: trialJobDetail.form.sequenceId.toString() },
             { key: 'MULTI_PHASE', value: this.isMultiPhase.toString() }
         ];
         if (gpuNum !== undefined) {
@@ -562,7 +536,7 @@ class LocalTrainingService implements TrainingService {
         const scriptName: string = getScriptName('run');
         await fs.promises.writeFile(path.join(trialJobDetail.workingDirectory, scriptName),
                                     runScriptContent.join(getNewLine()), { encoding: 'utf8', mode: 0o777 });
-        await this.writeParameterFile(trialJobDetail.workingDirectory, (<TrialJobApplicationForm>trialJobDetail.form).hyperParameters);
+        await this.writeParameterFile(trialJobDetail.workingDirectory, trialJobDetail.form.hyperParameters);
         const trialJobProcess: cp.ChildProcess = runScript(path.join(trialJobDetail.workingDirectory, scriptName));
         this.setTrialJobStatus(trialJobDetail, 'RUNNING');
         trialJobDetail.startTime = Date.now();
@@ -589,60 +563,10 @@ class LocalTrainingService implements TrainingService {
         this.jobStreamMap.set(trialJobDetail.id, stream);
     }
 
-    private async runHostJob(form: HostJobApplicationForm): Promise<TrialJobDetail> {
-        const jobId: string = uniqueString(5);
-        const workDir: string = path.join(this.rootDir, 'hostjobs', jobId);
-        await cpp.exec(`mkdir -p ${workDir}`);
-        const wrappedCmd: string = `cd ${workDir} && ${form.cmd}>stdout 2>stderr`;
-        this.log.debug(`runHostJob: command: ${wrappedCmd}`);
-        const process: cp.ChildProcess = cp.exec(wrappedCmd);
-        const jobDetail: LocalTrialJobDetail = {
-            id: jobId,
-            status: 'RUNNING',
-            submitTime: Date.now(),
-            workingDirectory: workDir,
-            form: form,
-            sequenceId: this.generateSequenceId(),
-            pid: process.pid
-        };
-        this.jobMap.set(jobId, jobDetail);
-        this.log.debug(`runHostJob: return: ${JSON.stringify(jobDetail)} `);
-
-        return jobDetail;
-    }
-
-    private async getHostJob(jobId: string): Promise<TrialJobDetail> {
-        const jobDetail: LocalTrialJobDetail | undefined = this.jobMap.get(jobId);
-        if (jobDetail === undefined) {
-            throw new NNIError(NNIErrorNames.NOT_FOUND, `Host Job not found: ${jobId}`);
-        }
-        try {
-            await cpp.exec(`kill -0 ${jobDetail.pid}`);
-
-            return jobDetail;
-        } catch (error) {
-            if (error instanceof Error) {
-                this.log.debug(`getHostJob: error: ${error.message}`);
-                this.jobMap.delete(jobId);
-                throw new NNIError(NNIErrorNames.NOT_FOUND, `Host Job not found: ${error.message}`);
-            } else {
-                throw error;
-            }
-        }
-    }
-
     private async writeParameterFile(directory: string, hyperParameters: HyperParameters): Promise<void> {
         const filepath: string = path.join(directory, generateParamFileName(hyperParameters));
         await fs.promises.writeFile(filepath, hyperParameters.value, { encoding: 'utf8' });
     }
-
-    private generateSequenceId(): number {
-        if (this.trialSequenceId === -1) {
-            this.trialSequenceId = getInitTrialSequenceId();
-        }
-
-        return this.trialSequenceId++;
-    }
 }
 
 export { LocalTrainingService };

src/nni_manager/training_service/pai/paiData.ts

@@ -19,7 +19,7 @@
 
 'use strict';
 
-import { JobApplicationForm, TrialJobDetail, TrialJobStatus  } from '../../common/trainingService';
+import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus  } from '../../common/trainingService';
 
 /**
  * PAI trial job detail
@@ -34,20 +34,18 @@ export class PAITrialJobDetail implements TrialJobDetail {
     public tags?: string[];
     public url?: string;
     public workingDirectory: string;
-    public form: JobApplicationForm;
-    public sequenceId: number;
+    public form: TrialJobApplicationForm;
     public hdfsLogPath: string;
     public isEarlyStopped?: boolean;
 
     constructor(id: string, status: TrialJobStatus, paiJobName : string,
-                submitTime: number, workingDirectory: string, form: JobApplicationForm, sequenceId: number, hdfsLogPath: string) {
+                submitTime: number, workingDirectory: string, form: TrialJobApplicationForm, hdfsLogPath: string) {
         this.id = id;
         this.status = status;
         this.paiJobName = paiJobName;
         this.submitTime = submitTime;
         this.workingDirectory = workingDirectory;
         this.form = form;
-        this.sequenceId = sequenceId;
         this.tags = [];
         this.hdfsLogPath = hdfsLogPath;
     }

src/nni_manager/training_service/pai/paiTrainingService.ts

@@ -30,10 +30,10 @@ import { EventEmitter } from 'events';
 import { Deferred } from 'ts-deferred';
 import { String } from 'typescript-string-operations';
 import { MethodNotImplementedError } from '../../common/errors';
-import { getExperimentId, getInitTrialSequenceId } from '../../common/experimentStartupInfo';
+import { getExperimentId } from '../../common/experimentStartupInfo';
 import { getLogger, Logger } from '../../common/log';
 import {
-    HyperParameters, JobApplicationForm, NNIManagerIpConfig, TrainingService,
+    HyperParameters, NNIManagerIpConfig, TrainingService,
     TrialJobApplicationForm, TrialJobDetail, TrialJobMetric
 } from '../../common/trainingService';
 import { delay, generateParamFileName,
@@ -70,7 +70,6 @@ class PAITrainingService implements TrainingService {
     private readonly paiTokenUpdateInterval: number;
     private readonly experimentId! : string;
     private readonly paiJobCollector : PAIJobInfoCollector;
-    private nextTrialSequenceId: number;
     private paiRestServerPort?: number;
     private nniManagerIpConfig?: NNIManagerIpConfig;
     private copyExpCodeDirPromise?: Promise<void>;
@@ -90,7 +89,6 @@ class PAITrainingService implements TrainingService {
         this.expRootDir = path.join('/nni', 'experiments', getExperimentId());
         this.experimentId = getExperimentId();
         this.paiJobCollector = new PAIJobInfoCollector(this.trialJobsMap);
-        this.nextTrialSequenceId = -1;
         this.paiTokenUpdateInterval = 7200000; //2hours
         this.logCollection = 'none';
         this.log.info('Construct OpenPAI training service.');
@@ -112,9 +110,7 @@ class PAITrainingService implements TrainingService {
         const jobs: TrialJobDetail[] = [];
 
         for (const [key, value] of this.trialJobsMap) {
-            if (value.form.jobType === 'TRIAL') {
-                jobs.push(await this.getTrialJob(key));
-            }
+            jobs.push(await this.getTrialJob(key));
         }
 
         return Promise.resolve(jobs);
@@ -142,7 +138,7 @@ class PAITrainingService implements TrainingService {
         this.metricsEmitter.off('metric', listener);
     }
 
-    public async submitTrialJob(form: JobApplicationForm): Promise<TrialJobDetail> {
+    public async submitTrialJob(form: TrialJobApplicationForm): Promise<TrialJobDetail> {
         if (this.paiClusterConfig === undefined) {
             throw new Error(`paiClusterConfig not initialized!`);
         }
@@ -151,7 +147,6 @@ class PAITrainingService implements TrainingService {
         this.log.info(`submitTrialJob: form: ${JSON.stringify(form)}`);
 
         const trialJobId: string = uniqueString(5);
-        const trialSequenceId: number = this.generateSequenceId();
         //TODO: use HDFS working folder instead
         const trialWorkingFolder: string = path.join(this.expRootDir, 'trials', trialJobId);
         const paiJobName: string = `nni_exp_${this.experimentId}_trial_${trialJobId}`;
@@ -171,7 +166,6 @@ class PAITrainingService implements TrainingService {
             Date.now(),
             trialWorkingFolder,
             form,
-            trialSequenceId,
             hdfsLogPath);
 
         this.trialJobsMap.set(trialJobId, trialJobDetail);
@@ -181,16 +175,12 @@ class PAITrainingService implements TrainingService {
         return deferred.promise;
     }
 
-    public async updateTrialJob(trialJobId: string, form: JobApplicationForm): Promise<TrialJobDetail> {
+    public async updateTrialJob(trialJobId: string, form: TrialJobApplicationForm): Promise<TrialJobDetail> {
         const trialJobDetail: undefined | TrialJobDetail = this.trialJobsMap.get(trialJobId);
         if (trialJobDetail === undefined) {
             throw new Error(`updateTrialJob failed: ${trialJobId} not found`);
         }
-        if (form.jobType === 'TRIAL') {
-                await this.writeParameterFile(trialJobId, (<TrialJobApplicationForm>form).hyperParameters);
-        } else {
-            throw new Error(`updateTrialJob failed: jobType ${form.jobType} not supported.`);
-        }
+        await this.writeParameterFile(trialJobId, form.hyperParameters);
 
         return trialJobDetail;
     }
@@ -397,11 +387,10 @@ class PAITrainingService implements TrainingService {
         await fs.promises.writeFile(path.join(trialLocalTempFolder, 'install_nni.sh'), runScriptContent, { encoding: 'utf8' });
 
         // Write file content ( parameter.cfg ) to local tmp folders
-        const trialForm : TrialJobApplicationForm = (<TrialJobApplicationForm>trialJobDetail.form);
-        if (trialForm !== undefined) {
+        if (trialJobDetail.form !== undefined) {
             await fs.promises.writeFile(
-                path.join(trialLocalTempFolder, generateParamFileName(trialForm.hyperParameters)),
-                trialForm.hyperParameters.value, { encoding: 'utf8' }
+                path.join(trialLocalTempFolder, generateParamFileName(trialJobDetail.form.hyperParameters)),
+                trialJobDetail.form.hyperParameters.value, { encoding: 'utf8' }
             );
         }
         const hdfsCodeDir: string = HDFSClientUtility.getHdfsTrialWorkDir(this.paiClusterConfig.userName, trialJobId);
@@ -416,7 +405,7 @@ class PAITrainingService implements TrainingService {
             `$PWD/${trialJobId}/nnioutput`,
             trialJobId,
             this.experimentId,
-            trialJobDetail.sequenceId,
+            trialJobDetail.form.sequenceId,
             this.isMultiPhase,
             this.paiTrialConfig.command,
             nniManagerIp,
@@ -507,14 +496,6 @@ class PAITrainingService implements TrainingService {
         return deferred.promise;
     }
 
-    private generateSequenceId(): number {
-        if (this.nextTrialSequenceId === -1) {
-            this.nextTrialSequenceId = getInitTrialSequenceId();
-        }
-
-        return this.nextTrialSequenceId++;
-    }
-
     private async statusCheckingLoop(): Promise<void> {
         while (!this.stopping) {
             try {

src/nni_manager/training_service/remote_machine/remoteMachineData.ts

@@ -22,7 +22,7 @@
 import * as fs from 'fs';
 import { Client, ConnectConfig } from 'ssh2';
 import { Deferred } from 'ts-deferred';
-import { JobApplicationForm, TrialJobDetail, TrialJobStatus  } from '../../common/trainingService';
+import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus  } from '../../common/trainingService';
 import { GPUInfo, GPUSummary } from '../common/gpuData';
 
 /**
@@ -82,20 +82,18 @@ export class RemoteMachineTrialJobDetail implements TrialJobDetail {
     public tags?: string[];
     public url?: string;
     public workingDirectory: string;
-    public form: JobApplicationForm;
-    public sequenceId: number;
+    public form: TrialJobApplicationForm;
     public rmMeta?: RemoteMachineMeta;
     public isEarlyStopped?: boolean;
     public gpuIndices: GPUInfo[];
 
     constructor(id: string, status: TrialJobStatus, submitTime: number,
-                workingDirectory: string, form: JobApplicationForm, sequenceId: number) {
+                workingDirectory: string, form: TrialJobApplicationForm) {
         this.id = id;
         this.status = status;
         this.submitTime = submitTime;
         this.workingDirectory = workingDirectory;
         this.form = form;
-        this.sequenceId = sequenceId;
         this.tags = [];
         this.gpuIndices = [];
     }