[ci] combine pip installs in Python test scripts (#6981)

* [ci] combine pip installs in Python test scripts

* remove most CI

* Revert "remove most CI"

This reverts commit 8c0c1ec7b6e4e9f69190965abbf58b760efc8c6d.

.ci/pip-envs/requirements-latest.txt

+# nightlies for: matplotlib, numpy, pandas, pyarrow, scipy, scikit-learn
+--extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple
+
+# runtime dependencies (using `.dev0` suffix to allow nightlies)
+#
+# pinning rules:
+#
+#   * latest versions of lightgbm's dependencies,
+#   * including pre-releases and nightlies
+#
+cffi>=1.17.1
+matplotlib>=3.11.0.dev0
+numpy>=2.4.0.dev0
+pandas>=3.0.0.dev0
+pyarrow>=22.0.0.dev0
+scikit-learn>=1.8.dev0
+scipy>=1.17.0.dev0
+
+# testing-only dependencies
+cloudpickle>=3.1.1
+psutil>=7.0
+pytest>=8.4.1

.ci/pip-envs/requirements-oldest.txt

+# oldest versions of dependencies published after
+# minimum supported Python version's first release,
+# for which there are wheels compatible with the
+# python:{version} image
+#
+# see https://devguide.python.org/versions/
+#
+cffi==1.15.1
+numpy==1.19.0
+pandas==1.1.3
+pyarrow==6.0.1
+scikit-learn==0.24.2
+scipy==1.6.0

.ci/test-python-latest.sh

@@ -2,44 +2,12 @@
 
 set -e -E -u -o pipefail
 
-# latest versions of lightgbm's dependencies,
-# including pre-releases and nightlies
-#
-# ref: https://github.com/pydata/xarray/blob/31111b3afe44fd6f7dac363264e94186cc5168d2/.github/workflows/upstream-dev-ci.yaml
-echo "installing testing dependencies"
-python -m pip install \
-    cloudpickle \
-    psutil \
-    pytest
-echo "done installing testing dependencies"
-
-echo "installing lightgbm's dependencies"
-python -m pip install \
-    --extra-index-url https://pypi.anaconda.org/scientific-python-nightly-wheels/simple \
-    --prefer-binary \
-    --pre \
-    --upgrade \
-        'numpy>=2.0.0.dev0' \
-        'matplotlib>=3.10.0.dev0' \
-        'pandas>=3.0.0.dev0' \
-        'scikit-learn>=1.6.dev0' \
-        'scipy>=1.15.0.dev0'
-
-python -m pip install \
-    --extra-index-url https://pypi.fury.io/arrow-nightlies/ \
+echo "installing lightgbm and its dependencies"
+pip install \
     --prefer-binary \
-    --pre \
     --upgrade \
-        'pyarrow>=17.0.0.dev0'
-
-python -m pip install \
-    'cffi>=1.15.1'
-
-echo "done installing lightgbm's dependencies"
-
-echo "installing lightgbm"
-pip install --no-deps dist/*.whl
-echo "done installing lightgbm"
+    -r ./.ci/pip-envs/requirements-latest.txt \
+    dist/*.whl
 
 echo "installed package versions:"
 pip freeze

.ci/test-python-oldest.sh

@@ -2,27 +2,12 @@
 
 set -e -E -u -o pipefail
 
-# oldest versions of dependencies published after
-# minimum supported Python version's first release,
-# for which there are wheels compatible with the
-# python:{version} image
-#
-# see https://devguide.python.org/versions/
-#
-echo "installing lightgbm's dependencies"
+echo "installing lightgbm and its dependencies"
 pip install \
-  'cffi==1.15.1' \
-  'numpy==1.19.0' \
-  'pandas==1.1.3' \
-  'pyarrow==6.0.1' \
-  'scikit-learn==0.24.2' \
-  'scipy==1.6.0' \
-|| exit 1
-echo "done installing lightgbm's dependencies"
-
-echo "installing lightgbm"
-pip install --no-deps dist/*.whl || exit 1
-echo "done installing lightgbm"
+    --prefer-binary \
+    --upgrade \
+    -r ./.ci/pip-envs/requirements-oldest.txt \
+    dist/*.whl
 
 echo "installed package versions:"
 pip freeze