Fix trial start time (#408)

* Fix trial job start time

* updates

* updates

src/nni_manager/common/datastore.ts

@@ -91,7 +91,7 @@ abstract class Database {
     public abstract queryExperimentProfile(experimentId: string, revision?: number): Promise<ExperimentProfile[]>;
     public abstract queryLatestExperimentProfile(experimentId: string): Promise<ExperimentProfile>;
     public abstract storeTrialJobEvent(
-        event: TrialJobEvent, trialJobId: string, hyperParameter?: string, jobDetail?: TrialJobDetail): Promise<void>;
+        event: TrialJobEvent, trialJobId: string, timestamp: number, hyperParameter?: string, jobDetail?: TrialJobDetail): Promise<void>;
     public abstract queryTrialJobEvent(trialJobId?: string, event?: TrialJobEvent): Promise<TrialJobEventRecord[]>;
     public abstract storeMetricData(trialJobId: string, data: string): Promise<void>;
     public abstract queryMetricData(trialJobId?: string, type?: MetricType): Promise<MetricDataRecord[]>;

src/nni_manager/core/nniDataStore.ts

@@ -87,7 +87,21 @@ class NNIDataStore implements DataStore {
         event: TrialJobEvent, trialJobId: string, hyperParameter?: string, jobDetail?: TrialJobDetail): Promise<void> {
         this.log.debug(`storeTrialJobEvent: event: ${event}, data: ${hyperParameter}, jobDetail: ${JSON.stringify(jobDetail)}`);
 
-        return this.db.storeTrialJobEvent(event, trialJobId, hyperParameter, jobDetail).catch(
+        // Use the timestamp in jobDetail as TrialJobEvent timestamp for different events
+        let timestamp: number | undefined;
+        if (event === 'WAITING' && jobDetail) {
+            timestamp = jobDetail.submitTime;
+        } else if (event === 'RUNNING' && jobDetail) {
+            timestamp = jobDetail.startTime;
+        } else if (['EARLY_STOPPED', 'SUCCEEDED', 'FAILED', 'USER_CANCELED', 'SYS_CANCELED'].includes(event) && jobDetail) {
+            timestamp = jobDetail.endTime;
+        }
+        // Use current time as timestamp if timestamp is not assigned from jobDetail
+        if (timestamp === undefined) {
+            timestamp = Date.now();
+        }
+
+        return this.db.storeTrialJobEvent(event, trialJobId, timestamp, hyperParameter, jobDetail).catch(
                 (err: Error) => {
                     throw new NNIError('Datastore error', `Datastore error: ${err.message}`, err);
                 }
@@ -272,6 +286,11 @@ class NNIDataStore implements DataStore {
                     if (record.logPath !== undefined) {
                         jobInfo.logPath = record.logPath;
                     }
+                    // Initially assign WAITING timestamp as job's start time,
+                    // If there is RUNNING state event, it will be updated as RUNNING state timestamp
+                    if (jobInfo.startTime === undefined && record.timestamp !== undefined) {
+                        jobInfo.startTime = record.timestamp;
+                    }
                     break;
                 case 'SUCCEEDED':
                 case 'FAILED':

src/nni_manager/core/sqlDatabase.ts

@@ -177,11 +177,11 @@ class SqlDB implements Database {
     }
 
     public storeTrialJobEvent(
-        event: TrialJobEvent, trialJobId: string, hyperParameter?: string, jobDetail?: TrialJobDetail): Promise<void> {
+        event: TrialJobEvent, trialJobId: string, timestamp: number, hyperParameter?: string, jobDetail?: TrialJobDetail): Promise<void> {
         const sql: string = 'insert into TrialJobEvent values (?,?,?,?,?,?)';
         const logPath: string | undefined = jobDetail === undefined ? undefined : jobDetail.url;
         const sequenceId: number | undefined = jobDetail === undefined ? undefined : jobDetail.sequenceId;
-        const args: any[] = [Date.now(), trialJobId, event, hyperParameter, logPath, sequenceId];
+        const args: any[] = [timestamp, trialJobId, event, hyperParameter, logPath, sequenceId];
 
         const deferred: Deferred<void> = new Deferred<void>();
         this.db.run(sql, args, (err: Error | null) => { this.resolve(deferred, err); });

src/nni_manager/core/test/sqlDatabase.test.ts

@@ -120,7 +120,7 @@ describe('core/sqlDatabase', () => {
             await (<SqlDB>db).storeExperimentProfile(profile);
         }
         for (const event of events) {
-            await (<SqlDB>db).storeTrialJobEvent(<TrialJobEvent>event.event, event.trialJobId, event.data);
+            await (<SqlDB>db).storeTrialJobEvent(<TrialJobEvent>event.event, event.trialJobId, Date.now(), event.data);
         }
         for (const metric of metrics) {
             await (<SqlDB>db).storeMetricData(metric.trialJobId, JSON.stringify(metric));