[Fix] Fix potential bug in using `analyze_logs` (#634)

* fix when killing before eval

* fix resume log

tools/analysis_tools/analyze_logs.py

@@ -53,10 +53,21 @@ def plot_curve(log_dicts, args):
                     f'{args.json_logs[i]} does not contain metric {metric}')
 
             if args.mode == 'eval':
-                xs = np.arange(args.interval, max(epochs) + 1, args.interval)
+                # if current training is resumed from previous checkpoint
+                # we lost information in early epochs
+                # `xs` should start according to `min(epochs)`
+                x0 = min(epochs) + args.interval - min(epochs) % args.interval
+                xs = np.arange(x0, max(epochs) + 1, args.interval)
                 ys = []
                 for epoch in epochs[args.interval - 1::args.interval]:
                     ys += log_dict[epoch][metric]
+
+                # if training is aborted before eval of the last epoch
+                # `xs` and `ys` will have different length and cause an error
+                # check if `ys[-1]` is empty here
+                if not log_dict[epoch][metric]:
+                    xs = xs[:-1]
+
                 ax = plt.gca()
                 ax.set_xticks(xs)
                 plt.xlabel('epoch')