trial.ts 12.6 KB
Newer Older
Lijiaoa's avatar
Lijiaoa committed
1
import * as JSON5 from 'json5';
2
3
4
5
6
7
8
9
import {
    MetricDataRecord,
    TrialJobInfo,
    TableObj,
    TableRecord,
    Parameters,
    FinalType,
    MultipleAxes,
10
11
    SingleAxis,
    RetiariiParameter
12
13
14
15
16
17
18
19
20
21
} from '../interface';
import {
    getFinal,
    formatAccuracy,
    metricAccuracy,
    parseMetrics,
    isArrayType,
    isNaNorInfinity,
    formatComplexTypeValue
} from '../function';
22

23
24
25
26
27
28
29
/**
 * Get a structured representation of parameters
 * @param paramObj Parameters object
 * @param space All axes from search space (or sub search space)
 * @param prefix Current namespace (to make full name for unexpected entries)
 * @returns Parsed structured parameters and unexpected entries
 */
30
31
32
33
34
function inferTrialParameters(
    paramObj: object,
    space: MultipleAxes,
    prefix: string = ''
): [Map<SingleAxis, any>, Map<string, any>] {
35
36
    const latestedParamObj =
        'mutation_summary' in paramObj ? (paramObj as RetiariiParameter).mutation_summary : paramObj;
37
38
    const parameters = new Map<SingleAxis, any>();
    const unexpectedEntries = new Map<string, any>();
39
    for (const [k, v] of Object.entries(latestedParamObj)) {
40
41
        // prefix can be a good fallback when corresponding item is not found in namespace
        const axisKey = space.axes.get(k);
42
        if (prefix && k === '_name') continue;
43
44
45
46
47
48
49
50
51
52
53
        if (axisKey !== undefined) {
            if (typeof v === 'object' && v._name !== undefined && axisKey.nested) {
                // nested entry
                parameters.set(axisKey, v._name);
                const subSpace = axisKey.domain.get(v._name);
                if (subSpace !== undefined) {
                    const [subParams, subUnexpected] = inferTrialParameters(v, subSpace, prefix + k + '/');
                    subParams.forEach((v, k) => parameters.set(k, v));
                    subUnexpected.forEach((v, k) => unexpectedEntries.set(k, v));
                }
            } else {
Lijiaoa's avatar
Lijiaoa committed
54
                parameters.set(axisKey, formatComplexTypeValue(v));
55
56
            }
        } else {
Lijiaoa's avatar
Lijiaoa committed
57
            unexpectedEntries.set(prefix + k, formatComplexTypeValue(v));
58
59
60
61
62
        }
    }
    return [parameters, unexpectedEntries];
}

63
64
65
class Trial implements TableObj {
    private metricsInitialized: boolean = false;
    private infoField: TrialJobInfo | undefined;
66
    public intermediates: (MetricDataRecord | undefined)[] = [];
67
    public final: MetricDataRecord | undefined;
68
69
70
71
72
73
74
75
76
77
78
79
80
    private finalAcc: number | undefined;

    constructor(info?: TrialJobInfo, metrics?: MetricDataRecord[]) {
        this.infoField = info;
        if (metrics) {
            this.updateMetrics(metrics);
        }
    }

    public compareAccuracy(otherTrial: Trial): number | undefined {
        if (!this.sortable || !otherTrial.sortable) {
            return undefined;
        }
Lijiao's avatar
Lijiao committed
81
        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
82
83
84
85
        return this.finalAcc! - otherTrial.finalAcc!;
    }

    get info(): TrialJobInfo {
Lijiao's avatar
Lijiao committed
86
        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
87
88
89
90
        return this.infoField!;
    }

    get intermediateMetrics(): MetricDataRecord[] {
91
        const ret: MetricDataRecord[] = [];
92
93
        for (let i = 0; i < this.intermediates.length; i++) {
            if (this.intermediates[i]) {
Lijiao's avatar
Lijiao committed
94
                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
95
96
97
98
99
100
101
102
103
104
105
106
107
                ret.push(this.intermediates[i]!);
            } else {
                break;
            }
        }
        return ret;
    }

    get accuracy(): number | undefined {
        return this.finalAcc;
    }

    get sortable(): boolean {
108
        return this.metricsInitialized && this.finalAcc !== undefined && isFinite(this.finalAcc);
109
110
    }

Lijiao's avatar
Lijiao committed
111
112
113
114
115
116
    get latestAccuracy(): number | undefined {
        if (this.accuracy !== undefined) {
            return this.accuracy;
        } else if (this.intermediates.length > 0) {
            const temp = this.intermediates[this.intermediates.length - 1];
            if (temp !== undefined) {
117
118
                if (isArrayType(parseMetrics(temp.data))) {
                    return undefined;
119
120
                } else if (
                    typeof parseMetrics(temp.data) === 'object' &&
121
                    // eslint-disable-next-line no-prototype-builtins
122
123
                    parseMetrics(temp.data).hasOwnProperty('default')
                ) {
124
                    return parseMetrics(temp.data).default;
125
                } else if (typeof parseMetrics(temp.data) === 'number') {
126
127
                    return parseMetrics(temp.data);
                }
Lijiao's avatar
Lijiao committed
128
129
130
131
132
133
134
            } else {
                return undefined;
            }
        } else {
            return undefined;
        }
    }
135
136
137
138
    /* table obj start */

    get tableRecord(): TableRecord {
        const endTime = this.info.endTime || new Date().getTime();
Lijiao's avatar
Lijiao committed
139
        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
140
        const duration = (endTime - this.info.startTime!) / 1000;
Lijiaoa's avatar
Lijiaoa committed
141
        let accuracy;
142
143
        if (this.acc !== undefined && this.acc.default !== undefined) {
            if (typeof this.acc.default === 'number') {
Lijiaoa's avatar
Lijiaoa committed
144
                accuracy = JSON5.parse(this.acc.default);
145
            } else {
Lijiaoa's avatar
Lijiaoa committed
146
147
148
                accuracy = this.acc.default;
            }
        }
149

150
        return {
J-shang's avatar
J-shang committed
151
            key: this.info.trialJobId,
152
            sequenceId: this.info.sequenceId,
J-shang's avatar
J-shang committed
153
            id: this.info.trialJobId,
Lijiao's avatar
Lijiao committed
154
            // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
155
156
157
158
            startTime: this.info.startTime!,
            endTime: this.info.endTime,
            duration,
            status: this.info.status,
159
            message: this.info.message || '--',
160
            intermediateCount: this.intermediates.length,
Lijiaoa's avatar
Lijiaoa committed
161
            accuracy: accuracy,
Lijiao's avatar
Lijiao committed
162
163
            latestAccuracy: this.latestAccuracy,
            formattedLatestAccuracy: this.formatLatestAccuracy(),
164
            accDictionary: this.acc
165
166
167
168
169
170
171
172
173
174
175
176
        };
    }

    get key(): number {
        return this.info.sequenceId;
    }

    get sequenceId(): number {
        return this.info.sequenceId;
    }

    get id(): string {
J-shang's avatar
J-shang committed
177
        return this.info.trialJobId;
178
179
180
181
    }

    get duration(): number {
        const endTime = this.info.endTime || new Date().getTime();
Lijiao's avatar
Lijiao committed
182
        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
183
184
185
186
187
188
189
190
        return (endTime - this.info.startTime!) / 1000;
    }

    get status(): string {
        return this.info.status;
    }

    get acc(): FinalType | undefined {
Lijiaoa's avatar
Lijiaoa committed
191
192
193
        if (this.info === undefined) {
            return undefined;
        }
194
195
196
197
        return getFinal(this.info.finalMetricData);
    }

    get description(): Parameters {
chicm-ms's avatar
chicm-ms committed
198
        const ret: Parameters = {
199
200
            parameters: {},
            intermediate: [],
201
202
203
204
205
206
207
208
209
210
211
212
            multiProgress: 1
        };
        const tempHyper = this.info.hyperParameters;
        if (tempHyper !== undefined) {
            const getPara = JSON.parse(tempHyper[tempHyper.length - 1]).parameters;
            ret.multiProgress = tempHyper.length;
            if (typeof getPara === 'string') {
                ret.parameters = JSON.parse(getPara);
            } else {
                ret.parameters = getPara;
            }
        } else {
213
            ret.parameters = { error: "This trial's parameters are not available." };
214
215
216
217
218
        }
        if (this.info.logPath !== undefined) {
            ret.logPath = this.info.logPath;
        }

219
        const mediate: number[] = [];
220
        for (const items of this.intermediateMetrics) {
chicm-ms's avatar
chicm-ms committed
221
222
            if (typeof parseMetrics(items.data) === 'object') {
                mediate.push(parseMetrics(items.data).default);
223
            } else {
chicm-ms's avatar
chicm-ms committed
224
                mediate.push(parseMetrics(items.data));
225
226
227
228
229
230
            }
        }
        ret.intermediate = mediate;
        return ret;
    }

231
    public parameters(axes: MultipleAxes): Map<SingleAxis, any> {
232
        const ret = new Map<SingleAxis, any>(Array.from(axes.axes.values()).map(k => [k, null]));
Lijiaoa's avatar
Lijiaoa committed
233
        if (this.info === undefined || this.info.hyperParameters === undefined) {
234
            throw ret;
235
        } else {
Lijiaoa's avatar
Lijiaoa committed
236
            const tempHyper = this.info.hyperParameters;
237
238
239
240
            let params = JSON.parse(tempHyper[tempHyper.length - 1]).parameters;
            if (typeof params === 'string') {
                params = JSON.parse(params);
            }
241
            const [updated, unexpectedEntries] = inferTrialParameters(params, axes);
242
243
244
            if (unexpectedEntries.size) {
                throw unexpectedEntries;
            }
245
246
247
248
            for (const [k, v] of updated) {
                ret.set(k, v);
            }
            return ret;
249
250
251
252
        }
    }

    public metrics(space: MultipleAxes): Map<SingleAxis, any> {
253
254
        // set default value: null
        const ret = new Map<SingleAxis, any>(Array.from(space.axes.values()).map(k => [k, null]));
255
256
257
258
259
260
261
262
        const unexpectedEntries = new Map<string, any>();
        if (this.acc === undefined) {
            return ret;
        }
        const acc = typeof this.acc === 'number' ? { default: this.acc } : this.acc;
        Object.entries(acc).forEach(item => {
            const [k, v] = item;
            const column = space.axes.get(k);
263

264
265
266
267
268
269
270
271
272
273
274
275
            if (column !== undefined) {
                ret.set(column, v);
            } else {
                unexpectedEntries.set(k, v);
            }
        });
        if (unexpectedEntries.size) {
            throw unexpectedEntries;
        }
        return ret;
    }

276
277
278
279
    get color(): string | undefined {
        return undefined;
    }

280
    public finalKeys(): string[] {
281
        if (this.acc !== undefined) {
Lijiaoa's avatar
Lijiaoa committed
282
283
284
285
            return Object.keys(this.acc);
        } else {
            return [];
        }
286
287
    }

288
289
290
    /* table obj end */

    public initialized(): boolean {
291
        return Boolean(this.infoField);
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
    }

    public updateMetrics(metrics: MetricDataRecord[]): boolean {
        // parameter `metrics` must contain all known metrics of this trial
        this.metricsInitialized = true;
        const prevMetricCnt = this.intermediates.length + (this.final ? 1 : 0);
        if (metrics.length <= prevMetricCnt) {
            return false;
        }
        for (const metric of metrics) {
            if (metric.type === 'PERIODICAL') {
                this.intermediates[metric.sequence] = metric;
            } else {
                this.final = metric;
                this.finalAcc = metricAccuracy(metric);
            }
        }
        return true;
    }

    public updateLatestMetrics(metrics: MetricDataRecord[]): boolean {
        // this method is effectively identical to `updateMetrics`, but has worse performance
        this.metricsInitialized = true;
        let updated = false;
        for (const metric of metrics) {
            if (metric.type === 'PERIODICAL') {
                updated = updated || !this.intermediates[metric.sequence];
                this.intermediates[metric.sequence] = metric;
            } else {
                updated = updated || !this.final;
                this.final = metric;
                this.finalAcc = metricAccuracy(metric);
            }
        }
        return updated;
    }

    public updateTrialJobInfo(trialJobInfo: TrialJobInfo): boolean {
330
        const same = this.infoField && this.infoField.status === trialJobInfo.status;
331
332
333
334
335
336
337
338
        this.infoField = trialJobInfo;
        if (trialJobInfo.finalMetricData) {
            this.final = trialJobInfo.finalMetricData[trialJobInfo.finalMetricData.length - 1];
            this.finalAcc = metricAccuracy(this.final);
        }
        return !same;
    }

Lijiaoa's avatar
Lijiaoa committed
339
    private renderNumber(val: any): string {
340
        if (typeof val === 'number') {
Lijiaoa's avatar
Lijiaoa committed
341
342
            if (isNaNorInfinity(val)) {
                return `${val}`; // show 'NaN' or 'Infinity'
343
            } else {
344
345
346
347
348
                if (this.accuracy === undefined) {
                    return `${formatAccuracy(val)} (LATEST)`;
                } else {
                    return `${formatAccuracy(val)} (FINAL)`;
                }
349
            }
350
        } else {
Lijiaoa's avatar
Lijiaoa committed
351
352
353
354
355
            // show other types, such as {tensor: {data: }}
            return JSON.stringify(val);
        }
    }

356
357
    public formatLatestAccuracy(): string {
        // TODO: this should be private
358
        if (this.status === 'SUCCEEDED') {
359
            return this.accuracy === undefined ? '--' : this.renderNumber(this.accuracy);
Lijiaoa's avatar
Lijiaoa committed
360
361
362
363
364
        } else {
            if (this.accuracy !== undefined) {
                return this.renderNumber(this.accuracy);
            } else if (this.intermediates.length === 0) {
                return '--';
365
            } else {
Lijiaoa's avatar
Lijiaoa committed
366
367
368
                // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
                const latest = this.intermediates[this.intermediates.length - 1]!;
                return this.renderNumber(metricAccuracy(latest));
369
            }
370
371
372
373
374
        }
    }
}

export { Trial };