[Misc] Abort previous CI. (#5346)

* Abort previous CI

* script

* newtry

* more

* fix

* comment

* add notes

---------
Co-authored-by: Ubuntu <ubuntu@ip-172-31-28-63.ap-northeast-1.compute.internal>

Jenkinsfile

@@ -237,6 +237,22 @@ pipeline {
     stage('CI') {
       when { expression { !regression_test_done } }
       stages {
+        stage('Abort Previous CI') {
+          steps {
+            script {
+              // Jenkins will abort an older build if a newer build already
+              // passed a higher milestone.
+              // https://www.jenkins.io/doc/pipeline/steps/pipeline-milestone-step/
+              // Note: incorrect "success" status might be shown for the old
+              // runs in the PR.
+              def buildNumber = env.BUILD_NUMBER as int
+              for (int i = 1; i <= buildNumber; i++) {
+                milestone(i)
+              }
+            }
+          }
+        }
+
         stage('Lint Check') {
           agent {
             docker {

tests/scripts/ci_report/status.py

@@ -26,6 +26,9 @@ Build ID: {BUILD_ID}\n
 Status: {status_output}\n
 Report path: [link](https://dgl-ci-result.s3.us-west-2.amazonaws.com/{JOB_NAME}/{BUILD_NUMBER}/{BUILD_ID}/logs/report.html)\n
 Full logs path: [link](https://dgl-ci-result.s3.us-west-2.amazonaws.com/{JOB_NAME}/{BUILD_NUMBER}/{BUILD_ID}/logs/cireport.log)
+Note: A new CI run will cancel previous CI runs, but an incorrect "success"
+status might be shown for the previous runs. Please double check the report
+before merging the PR.
 """
 
 print(comment)