[Benchmark] Fix benchmark tests and add script to generate excel result (#2681)

* add script

* 21

* log

* fix regression tests

* add iter time

* fix

* Revert "fix"

This reverts commit 9b4587ad61dfeb82dcce6d1de188809f62947663.

* fix

* fix

* add

* add ogb instruction

* add cu11 dockerfile

* fix

* fix

* more iter apply_edge

* add iteration time
Co-authored-by: Minjie Wang <wmjlyjemaine@gmail.com>

benchmarks/README.md

@@ -35,6 +35,13 @@ To select which benchmark to run, use the `--bench` flag. For example,
 asv run -n -e --python=same --verbose --bench model_acc.bench_gat
 ```
 
+Note that OGB dataset need to be download manually to `/tmp/dataset` folder (i.e. `/tmp/dataset/ogbn-products/`) beforehand. 
+You can do it by runnnig the code below in this folder
+```python
+from benchmarks.utils import get_ogb_graph
+get_ogb_graph("ogbn-product")
+```
+
 Run in docker locally
 ---
 

benchmarks/benchmarks/api/bench_builtin_apply_edges.py

@@ -25,12 +25,13 @@ def track_time(graph_name, format, feat_size, reduce_type):
     }
 
     # dry run
+    for i in range(3):
         graph.apply_edges(reduce_builtin_dict[reduce_type])
 
     # timing
     
     with utils.Timer() as t:
-        for i in range(3):
+        for i in range(10):
             graph.apply_edges(reduce_builtin_dict[reduce_type])
 
-    return t.elapsed_secs / 3
+    return t.elapsed_secs / 10

benchmarks/benchmarks/api/bench_builtin_update_all_csc.py

@@ -34,14 +34,16 @@ def track_time(graph_name, format, feat_size, msg_type, reduce_type):
     }
 
     # dry run
+    
+    for i in range(3):
         graph.update_all(msg_builtin_dict[msg_type],
                          reduce_builtin_dict[reduce_type])
 
     # timing
 
     with utils.Timer() as t:
-        for i in range(3):
+        for i in range(10):
             graph.update_all(
                 msg_builtin_dict[msg_type], reduce_builtin_dict[reduce_type])
 
-    return t.elapsed_secs / 3
+    return t.elapsed_secs / 10

benchmarks/benchmarks/api/bench_knn_graph.py

@@ -12,13 +12,14 @@ from .. import utils
 @utils.parametrize('dim', [16, 128, 512])
 def track_time(size, dim, k):
     device = utils.get_bench_device()
-    features = np.random.randn(size, dim)
+    features = np.random.RandomState(42).randn(size, dim)
     feat = torch.tensor(features, dtype=torch.float, device=device)
     # dry run
+    for i in range(3):
         dgl.knn_graph(feat, k)
     # timing
     with utils.Timer() as t:
-        for i in range(10):
+        for i in range(20):
             dgl.knn_graph(feat, k)
 
-    return t.elapsed_secs / 10
+    return t.elapsed_secs / 20

benchmarks/benchmarks/api/bench_readout.py

@@ -19,15 +19,19 @@ def track_time(batch_size, feat_size, readout_op, type):
     g = dgl.batch(graphs).to(device)
     if type == 'node':
         g.ndata['h'] = torch.randn((g.num_nodes(), feat_size), device=device)
-        with utils.Timer() as t:
         for i in range(10):
             out = dgl.readout_nodes(g, 'h', op=readout_op)
+        with utils.Timer() as t:
+            for i in range(50):
+                out = dgl.readout_nodes(g, 'h', op=readout_op)
     elif type == 'edge':
         g.edata['h'] = torch.randn((g.num_edges(), feat_size), device=device)
-        with utils.Timer() as t:
         for i in range(10):
             out = dgl.readout_edges(g, 'h', op=readout_op)
+        with utils.Timer() as t:
+            for i in range(50):
+                out = dgl.readout_edges(g, 'h', op=readout_op)
     else:
         raise Exception("Unknown type")
 
-    return t.elapsed_secs / 10
+    return t.elapsed_secs / 50

benchmarks/benchmarks/api/bench_reverse.py

@@ -20,7 +20,7 @@ def track_time(graph_name, format):
 
     # timing
     with utils.Timer() as t:
-        for i in range(10):
+        for i in range(100):
             gg = dgl.reverse(graph)
 
-    return t.elapsed_secs / 10
+    return t.elapsed_secs / 100

benchmarks/benchmarks/api/bench_sample_neighbors.py

@@ -28,8 +28,8 @@ def track_time(graph_name, format, seed_nodes_num, fanout):
 
     # timing
     with utils.Timer() as t:
-        for i in range(3):
+        for i in range(50):
             dgl.sampling.sample_neighbors(
                 graph, seed_nodes, fanout, edge_dir=edge_dir)
 
-    return t.elapsed_secs / 3
+    return t.elapsed_secs / 50

benchmarks/scripts/generate_excel.py

+import pandas as pd
+import json
+from pathlib import Path
+from itertools import product
+
+
+def get_branch_name_from_hash(hash):
+    import subprocess
+    process = subprocess.Popen(['git', 'name-rev', '--name-only', hash],
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+    stdout, stderr = process.communicate()
+    if len(stderr) > 0:
+        return hash[:10]
+    else:
+        return stdout.decode("utf-8") .strip("\n")
+
+
+def main():
+    results_path = Path("../results")
+    results_path.is_dir()
+    benchmark_json_path = results_path / "benchmarks.json"
+    with benchmark_json_path.open() as f:
+        benchmark_json = json.load(f)
+    machines = [f for f in results_path.glob("*") if f.is_dir()]
+    output_results_dict = {}
+    for machine in machines:
+        # commit_results_dict = {}
+        per_machine_result = {}
+        commit_results_json_paths = [
+            f for f in machine.glob("*") if f.name != "machine.json"]
+        for commit in commit_results_json_paths:
+            with commit.open() as f:
+                commit_result = json.load(f)
+            commit_hash = commit_result['commit_hash']
+            per_commit_result = {}
+            for test_name, result in commit_result['results'].items():
+                per_commit_result[test_name] = []
+                if result['result'] is None:
+                    for test_args in product(*result['params']):
+                        per_commit_result[test_name].append(
+                            {"params": ", ".join(test_args), "result": None})
+                else:
+                    for test_args, performance_number in zip(product(*result['params']), result['result']):
+                        per_commit_result[test_name].append(
+                            {"params": ", ".join(test_args), "result": performance_number})
+            per_machine_result[commit_hash] = per_commit_result
+        output_results_dict[machine.name] = per_machine_result
+    return output_results_dict
+
+
+def dict_to_csv(output_results_dict):
+    with open("../results/benchmarks.json") as f:
+        benchmark_conf = json.load(f)
+    unit_dict = {}
+    for k, v in benchmark_conf.items():
+        if k != 'version':
+            unit_dict[k] = v['unit']
+    result_list = []
+    for machine, per_machine_result in output_results_dict.items():
+        for commit, test_cases in per_machine_result.items():
+            branch_name = get_branch_name_from_hash(commit)
+            result_column_name = "number_{}".format(branch_name)
+            # per_commit_result_list = []
+            for test_case_name, results in test_cases.items():
+                for result in results:
+                    result_list.append(
+                        {"test_name": test_case_name, 'params': result['params'], 'unit': unit_dict[test_case_name], "number": result['result'], 'commit': branch_name, 'machine': machine})
+    df = pd.DataFrame(result_list)
+    return df
+
+
+def side_by_side_view(df):
+    commits = c['commit'].unique().tolist()
+    full_df = df.loc[df['commit'] == commits[0]]
+    for commit in commits[1:]:
+        per_commit_df = df.loc[df['commit'] == commit]
+        full_df: pd.DataFrame = full_df.merge(
+            per_commit_df, on=['test_name', 'params', 'machine', 'unit'], how='outer', suffixes=("_{}".format(full_df.iloc[0]["commit"]), "_{}".format(per_commit_df.iloc[0]["commit"])))
+    full_df = full_df.loc[:, ~full_df.columns.str.startswith('commit')]
+    return full_df
+
+
+output_results_dict = main()
+df = dict_to_csv(output_results_dict)
+sbs_df = side_by_side_view(df)
+sbs_df.to_excel("result.xlsx")

docker/Dockerfile.ci_gpu_cu11

+# CI docker GPU env
+FROM nvidia/cuda:11.2.1-cudnn8-devel-ubuntu16.04
+
+RUN apt-get update --fix-missing
+
+COPY install/ubuntu_install_core.sh /install/ubuntu_install_core.sh
+RUN bash /install/ubuntu_install_core.sh
+
+COPY install/ubuntu_install_build.sh /install/ubuntu_install_build.sh
+RUN bash /install/ubuntu_install_build.sh
+
+# python
+COPY install/ubuntu_install_conda.sh /install/ubuntu_install_conda.sh
+RUN bash /install/ubuntu_install_conda.sh
+
+ENV CONDA_ALWAYS_YES="true"
+
+COPY install/conda_env/torch_gpu.yml /install/conda_env/torch_gpu.yml
+COPY install/conda_env/torch_gpu_pip_latest.txt /install/conda_env/torch_gpu_pip.txt
+RUN ["/bin/bash", "-i", "-c", "conda env create -f /install/conda_env/torch_gpu.yml"]
+
+# COPY install/conda_env/tensorflow_gpu.yml /install/conda_env/tensorflow_gpu.yml
+# RUN ["/bin/bash", "-i", "-c", "conda env create -f /install/conda_env/tensorflow_gpu.yml"]
+
+# COPY install/conda_env/mxnet_gpu.yml /install/conda_env/mxnet_gpu.yml
+# RUN ["/bin/bash", "-i", "-c", "conda env create -f /install/conda_env/mxnet_gpu.yml"]
+
+ENV CONDA_ALWAYS_YES=
+
+# Environment variables
+ENV PATH=/usr/local/nvidia/bin:${PATH}
+ENV PATH=/usr/local/cuda/bin:${PATH}
+ENV CPLUS_INCLUDE_PATH=/usr/local/cuda/include:${CPLUS_INCLUDE_PATH}
+ENV C_INCLUDE_PATH=/usr/local/cuda/include:${C_INCLUDE_PATH}
+ENV LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LIBRARY_PATH}
+ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}
+ENV CUDA_VISIBLE_DEVICES=0
+ENV TF_FORCE_GPU_ALLOW_GROWTH=true
\ No newline at end of file

docker/install/conda_env/torch_gpu_pip_latest.txt

+--find-links https://download.pytorch.org/whl/torch_stable.html
+torch==1.7.1+cu110
+torchvision==0.8.2+cu110
+pytest
+nose
+numpy
+cython
+scipy
+networkx
+matplotlib
+nltk
+requests[security]
+tqdm  
\ No newline at end of file