[CI] pre-install required py packages in image for unit/tutorial test (#5276)

Jenkinsfile

@@ -262,7 +262,7 @@ pipeline {
               agent {
                 docker {
                   label "linux-cpu-node"
-                  image "dgllib/dgl-ci-cpu:v221216"
+                  image "dgllib/dgl-ci-cpu:v230210"
                   args "-u root"
                   alwaysPull true
                 }
@@ -280,7 +280,7 @@ pipeline {
               agent {
                 docker {
                   label "linux-cpu-node"
-                  image "dgllib/dgl-ci-gpu:cu102_v221216"
+                  image "dgllib/dgl-ci-gpu:cu102_v230210"
                   args "-u root"
                   alwaysPull true
                 }
@@ -335,7 +335,7 @@ pipeline {
               agent {
                 docker {
                   label "linux-cpu-node"
-                  image "dgllib/dgl-ci-cpu:v221216"
+                  image "dgllib/dgl-ci-cpu:v230210"
                   alwaysPull true
                 }
               }
@@ -352,7 +352,7 @@ pipeline {
               agent {
                 docker {
                   label "linux-gpu-node"
-                  image "dgllib/dgl-ci-gpu:cu102_v221216"
+                  image "dgllib/dgl-ci-gpu:cu102_v230210"
                   args "--runtime nvidia"
                   alwaysPull true
                 }
@@ -381,7 +381,7 @@ pipeline {
               agent {
                 docker {
                   label "linux-cpu-node"
-                  image "dgllib/dgl-ci-cpu:v220816"
+                  image "dgllib/dgl-ci-cpu:v230210"
                   alwaysPull true
                 }
               }
@@ -402,7 +402,7 @@ pipeline {
               agent {
                 docker {
                   label "linux-gpu-node"
-                  image "dgllib/dgl-ci-gpu:cu101_v220816"
+                  image "dgllib/dgl-ci-gpu:cu101_v230210"
                   args "--runtime nvidia"
                   alwaysPull true
                 }
@@ -424,7 +424,7 @@ pipeline {
               agent {
                 docker {
                   label "linux-cpu-node"
-                  image "dgllib/dgl-ci-cpu:v221216"
+                  image "dgllib/dgl-ci-cpu:v230210"
                   args "--shm-size=4gb"
                   alwaysPull true
                 }
@@ -476,7 +476,7 @@ pipeline {
               agent {
                 docker {
                   label "linux-gpu-node"
-                  image "dgllib/dgl-ci-gpu:cu102_v221216"
+                  image "dgllib/dgl-ci-gpu:cu102_v230210"
                   args "--runtime nvidia --shm-size=8gb"
                   alwaysPull true
                 }
@@ -504,7 +504,7 @@ pipeline {
               agent {
                 docker {
                   label "linux-cpu-node"
-                  image "dgllib/dgl-ci-cpu:v221216"
+                  image "dgllib/dgl-ci-cpu:v230210"
                   args "--shm-size=4gb"
                   alwaysPull true
                 }
@@ -549,7 +549,7 @@ pipeline {
               agent {
                 docker {
                   label "linux-cpu-node"
-                  image "dgllib/dgl-ci-cpu:v221216"
+                  image "dgllib/dgl-ci-cpu:v230210"
                   alwaysPull true
                 }
               }

docker/install/conda_env/mxnet_cpu.yml

@@ -14,3 +14,9 @@ dependencies:
     - nltk
     - requests[security]
     - tqdm
+    - psutil
+    - pyyaml
+    - pydantic
+    - pandas
+    - rdflib
+    - ogb

docker/install/conda_env/mxnet_gpu.yml

@@ -14,3 +14,9 @@ dependencies:
     - nltk
     - requests[security]
     - tqdm
+    - psutil
+    - pyyaml
+    - pydantic
+    - pandas
+    - rdflib
+    - ogb

docker/install/conda_env/tensorflow_cpu.yml

@@ -14,3 +14,9 @@ dependencies:
     - nltk
     - requests[security]
     - tqdm
+    - psutil
+    - pyyaml
+    - pydantic
+    - pandas
+    - rdflib
+    - ogb

docker/install/conda_env/tensorflow_gpu.yml

@@ -14,3 +14,9 @@ dependencies:
     - nltk
     - requests[security]
     - tqdm
+    - psutil
+    - pyyaml
+    - pydantic
+    - pandas
+    - rdflib
+    - ogb

docker/install/conda_env/torch_cpu_pip.txt

 --find-links https://download.pytorch.org/whl/torch_stable.html
 torch==1.12.0+cpu
+torchmetrics
 pytest
 nose
 numpy
@@ -13,3 +14,9 @@ tqdm
 pandas
 scikit-learn
 ogb
+psutil
+pyyaml
+pydantic
+rdflib
+filelock
+pyarrow

docker/install/conda_env/torch_gpu_pip.txt

 --find-links https://download.pytorch.org/whl/torch_stable.html
 torch==1.12.0+cu102
+torchmetrics
 pytest
 nose
 numpy
@@ -13,3 +14,7 @@ tqdm
 pandas
 scikit-learn
 ogb
+psutil
+pyyaml
+pydantic
+rdflib

docker/install/ubuntu_install_conda.sh

@@ -10,7 +10,7 @@ apt-get update --fix-missing && \
 wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh && \
     /bin/bash ~/miniconda.sh -b -p /opt/conda && \
     rm ~/miniconda.sh && \
-    /opt/conda/bin/conda clean -tipsy && \
+    /opt/conda/bin/conda clean -tipy && \
     ln -s /opt/conda/etc/profile.d/conda.sh /etc/profile.d/conda.sh && \
     echo ". /opt/conda/etc/profile.d/conda.sh" >> ~/.bashrc && \
     echo "conda activate base" >> ~/.bashrc

tests/scripts/task_distributed_test.sh

@@ -30,8 +30,6 @@ export CUDA_VISIBLE_DEVICES=-1
 
 conda activate ${DGLBACKEND}-ci
 
-python3 -m pip install pytest psutil pyyaml pydantic pandas rdflib ogb filelock pyarrow || fail "pip install"
-
 export PYTHONUNBUFFERED=1
 export OMP_NUM_THREADS=1
 export DMLC_LOG_DEBUG=1

tests/scripts/task_go_test.sh

@@ -23,5 +23,4 @@ popd
 export LC_ALL=C.UTF-8
 export LANG=C.UTF-8
 
-python -m pip install psutil || fail "pip install"
 python3 -m pytest -v --junitxml=pytest_go.xml --durations=100 tests/go/test_model.py || fail "go"

tests/scripts/task_mxnet_tutorial_test.sh

-#!/bin/bash
-# The working directory for this script will be "tests/scripts"
-
-. /opt/conda/etc/profile.d/conda.sh
-conda activate mxnet-ci
-TUTORIAL_ROOT="./tutorials"
-
-function fail {
-    echo FAIL: $@
-    exit -1
-}
-
-export MPLBACKEND=Agg
-export DGLBACKEND=mxnet
-export DGL_LIBRARY_PATH=${PWD}/build
-export PYTHONPATH=${PWD}/python:$PYTHONPATH
-export DGL_DOWNLOAD_DIR=${PWD}
-
-pushd ${TUTORIAL_ROOT} > /dev/null
-# Install requirements
-pip3 install -r requirements.txt || fail "installing requirements"
-
-# Test
-for f in $(find . -name "*_mx.py")
-do
-    echo "Running tutorial ${f} ..."
-    python3 $f || fail "run ${f}"
-done
-
-popd > /dev/null
\ No newline at end of file

tests/scripts/task_unit_test.sh

@@ -33,7 +33,6 @@ fi
 
 conda activate ${DGLBACKEND}-ci
 
-python3 -m pip install pytest psutil pyyaml pydantic pandas rdflib ogb torchmetrics || fail "pip install"
 if [ $DGLBACKEND == "mxnet" ]
 then
   python3 -m pytest -v --junitxml=pytest_compute.xml --durations=100 --ignore=tests/compute/test_ffi.py tests/compute || fail "compute"