[Feature] Try upload report to s3 (#3891)

refactor CI report and log

Jenkinsfile

@@ -304,7 +304,7 @@ pipeline {
                 }
               }
               stages {
-                stage('Unit test') {
+                stage('Tensorflow CPU Unit test') {
                   steps {
                     unit_test_linux('tensorflow', 'cpu')
                   }
@@ -326,7 +326,7 @@ pipeline {
                 }
               }
               stages {
-                stage('Unit test') {
+                stage('Tensorflow GPU Unit test') {
                   steps {
                     unit_test_linux('tensorflow', 'gpu')
                   }
@@ -348,17 +348,17 @@ pipeline {
                 }
               }
               stages {
-                stage('Unit test') {
+                stage('Torch CPU Unit test') {
                   steps {
                     unit_test_linux('pytorch', 'cpu')
                   }
                 }
-                stage('Example test') {
+                stage('Torch CPU Example test') {
                   steps {
                     example_test_linux('pytorch', 'cpu')
                   }
                 }
-                stage('Tutorial test') {
+                stage('Torch CPU Tutorial test') {
                   steps {
                     tutorial_test_linux('pytorch')
                   }
@@ -373,12 +373,12 @@ pipeline {
             stage('Torch CPU (Win64)') {
               agent { label 'windows' }
               stages {
-                stage('Unit test') {
+                stage('Torch CPU (Win64) Unit test') {
                   steps {
                     unit_test_win64('pytorch', 'cpu')
                   }
                 }
-                stage('Example test') {
+                stage('Torch CPU (Win64) Example test') {
                   steps {
                     example_test_win64('pytorch', 'cpu')
                   }
@@ -400,13 +400,13 @@ pipeline {
                 }
               }
               stages {
-                stage('Unit test') {
+                stage('Torch GPU Unit test') {
                   steps {
                     sh 'nvidia-smi'
                     unit_test_linux('pytorch', 'gpu')
                   }
                 }
-                stage('Example test') {
+                stage('Torch GPU Example test') {
                   steps {
                     example_test_linux('pytorch', 'gpu')
                   }
@@ -427,7 +427,7 @@ pipeline {
                 }
               }
               stages {
-                stage('Unit test') {
+                stage('MXNet CPU Unit test') {
                   steps {
                     unit_test_linux('mxnet', 'cpu')
                   }
@@ -454,7 +454,7 @@ pipeline {
                 }
               }
               stages {
-                stage('Unit test') {
+                stage('MXNet GPU Unit test') {
                   steps {
                     sh 'nvidia-smi'
                     unit_test_linux('mxnet', 'gpu')
@@ -474,8 +474,29 @@ pipeline {
   }
   post {
     always {
+      script {
+        node("linux-core-worker") {
+          docker.image('dgllib/dgl-ci-awscli:v220418').inside("--pull always --entrypoint=''") {
+            sh("rm -rf ci_tmp")
+            dir('ci_tmp') {
+              sh("curl -o cireport.log ${BUILD_URL}consoleText")
+              sh("curl -o report.py https://dgl-ci-scripts.s3.us-west-2.amazonaws.com/scripts/report.py")
+              sh("curl -o status.py https://dgl-ci-scripts.s3.us-west-2.amazonaws.com/scripts/status.py")
+              sh("curl -L ${BUILD_URL}wfapi")
+              sh("cat status.py")
+              sh("pytest --html=report.html --self-contained-html report.py || true")
+              sh("aws s3 sync ./ s3://dgl-ci-result/${JOB_NAME}/${BUILD_NUMBER}/${BUILD_ID}/logs/  --exclude '*' --include '*.log' --acl public-read --content-type text/plain")
+              sh("aws s3 sync ./ s3://dgl-ci-result/${JOB_NAME}/${BUILD_NUMBER}/${BUILD_ID}/logs/  --exclude '*.log' --acl public-read")
+
+              def comment = sh(returnStdout: true, script: "python3 status.py").trim()
+              echo(comment)
+              pullRequest.comment(comment)
+            }
+          }
+        }
         node('windows') {
-        bat "rmvirtualenv ${BUILD_TAG}"
+            bat(script: "rmvirtualenv ${BUILD_TAG}", returnStatus: true)
+        }
       }
     }
   }

docker/Dockerfile.awscli

+# Using the Ubuntu image (our OS)
+FROM ubuntu:latest
+# Update package manager (apt-get) 
+# and install (with the yes flag `-y`)
+# Python and Pip
+RUN apt-get update && apt-get install -y \
+    python3.8 \
+    python3-pip
+
+RUN apt-get install -y \
+        unzip \
+        curl \
+    && apt-get clean \
+    && curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" \
+    && unzip awscliv2.zip \
+    && ./aws/install \
+    && rm -rf \
+        awscliv2.zip
+
+RUN pip install pytest pytest-html requests

tests/scripts/ci_report/report.py

+from urllib.parse import urlparse, urljoin
+import os
+import requests
+import pytest
+import json
+import enum
+from pathlib import Path
+import tempfile
+
+
+class JobStatus(enum.Enum):
+    SUCCESS = 0
+    FAIL = 1
+    SKIP = 2
+
+
+JENKINS_STATUS_MAPPING = {
+    "SUCCESS": JobStatus.SUCCESS,
+    "ABORTED": JobStatus.FAIL,
+    "FAILED": JobStatus.FAIL,
+    "IN_PROGRESS": JobStatus.FAIL,
+    "NOT_EXECUTED": JobStatus.SKIP,
+    "PAUSED_PENDING_INPUT": JobStatus.SKIP,
+    "QUEUED": JobStatus.SKIP,
+    "UNSTABLE": JobStatus.FAIL,
+}
+
+assert "BUILD_URL" in os.environ, "Are you in the Jenkins environment?"
+job_link = os.environ["BUILD_URL"]
+response = requests.get('{}wfapi'.format(job_link)).json()
+domain = '{uri.scheme}://{uri.netloc}/'.format(uri=urlparse(job_link))
+stages = response["stages"]
+
+final_dict = {}
+failed_nodes = []
+nodes_dict = {}
+
+
+def get_jenkins_json(path):
+    return requests.get(urljoin(domain, path)).json()
+
+
+for stage in stages:
+    link = stage['_links']['self']['href']
+    stage_name = stage['name']
+    res = requests.get(urljoin(domain, link)).json()
+    nodes = res['stageFlowNodes']
+    for node in nodes:
+        nodes_dict[node['id']] = node
+        nodes_dict[node['id']]['stageName'] = stage_name
+
+def get_node_full_name(node, node_dict):
+    name = ""
+    while "parentNodes" in node:
+        name = name + "/" + node["name"]
+        id = node['parentNodes'][0]
+        if id in nodes_dict:
+            node = node_dict[id]
+        else:
+            break
+    return name
+
+for key, node in nodes_dict.items():
+    logs = get_jenkins_json(
+        node['_links']['log']['href']).get('text', '')
+    node_name = node['name']
+    if "Post Actions" in node['stageName']:
+        continue
+    node_status = node['status']
+    id = node['id']
+    full_name = get_node_full_name(node, nodes_dict)
+    final_dict["{}_{}/{}".format(id, node['stageName'], full_name)] = {
+        "status": JENKINS_STATUS_MAPPING[node_status],
+        "logs": logs
+    }
+
+JOB_NAME = os.getenv("JOB_NAME")
+BUILD_NUMBER = os.getenv("BUILD_NUMBER")
+BUILD_ID = os.getenv("BUILD_ID")
+
+prefix = f"https://dgl-ci-result.s3.us-west-2.amazonaws.com/{JOB_NAME}/{BUILD_NUMBER}/{BUILD_ID}/logs/logs_dir/"
+
+
+@pytest.mark.parametrize("test_name", final_dict)
+def test_generate_report(test_name):
+    os.makedirs("./logs_dir/", exist_ok=True)
+    tmp = tempfile.NamedTemporaryFile(
+        mode='w', delete=False, suffix=".log", dir="./logs_dir/")
+    tmp.write(final_dict[test_name]["logs"])
+    filename = Path(tmp.name).name
+    # print(final_dict[test_name]["logs"])
+    print("Log path: {}".format(prefix+filename))
+
+    if final_dict[test_name]["status"] == JobStatus.FAIL:
+        pytest.fail(
+            "Test failed. Please see the log at {}".format(prefix+filename))
+    elif final_dict[test_name]["status"] == JobStatus.SKIP:
+        pytest.skip(
+            "Test skipped. Please see the log at {}".format(prefix+filename))

tests/scripts/ci_report/status.py

+import os
+import requests
+JOB_NAME = os.getenv("JOB_NAME")
+BUILD_NUMBER = os.getenv("BUILD_NUMBER")
+BUILD_ID = os.getenv("BUILD_ID")
+
+
+job_link = os.environ["BUILD_URL"]
+response = requests.get('{}wfapi'.format(job_link)).json()
+# IN_PROGRESS is expected, sincce the pipeline is not finished when executing this script
+success = response["status"] in ['SUCCESS', 'IN_PROGRESS']
+
+if success:
+    status = "✅ CI test succeeded"
+else:
+    status = "❌ CI test failed"
+
+
+comment = f""" {JOB_NAME}
+{status} \n
+Report at [link](https://dgl-ci-result.s3.us-west-2.amazonaws.com/{JOB_NAME}/{BUILD_NUMBER}/{BUILD_ID}/logs/report.html) \n
+Full logs at [link](https://dgl-ci-result.s3.us-west-2.amazonaws.com/{JOB_NAME}/{BUILD_NUMBER}/{BUILD_ID}/logs/cireport.log)
+"""
+
+print(comment)
\ No newline at end of file