[ci][docs] use miniforge for readthedocs builds (fixes #4954) (#4957)

* [ci] [docs] use mamba for readthedocs builds (fixes #4954)

* update docs

* simplify build script and add docs flag to gitignore

* exit with non-0 if build fails

* update CI job

* add doxygen

* remove outdated requirement_base.txt reference

* use conda create instead of conda env create

* fix conda create flags

* add nodefaults to env.yml

* Update docs/README.rst
Co-authored-by: Nikita Titov <nekit94-08@mail.ru>

* try to fix check-docs CI job

* additional changes

* switch from mamba to miniforge

* simplify docker command and fix issues in local build script

* Apply suggestions from code review
Co-authored-by: Nikita Titov <nekit94-08@mail.ru>

* update docs and conda

* Apply suggestions from code review
Co-authored-by: Nikita Titov <nekit94-08@mail.ru>
Co-authored-by: Nikita Titov <nekit94-08@mail.ru>

.ci/test.sh

@@ -36,8 +36,15 @@ cd $BUILD_DIRECTORY
 
 if [[ $TASK == "check-docs" ]] || [[ $TASK == "check-links" ]]; then
     cd $BUILD_DIRECTORY/docs
-    conda install -q -y -n $CONDA_ENV -c conda-forge doxygen rstcheck
-    pip install --user -r requirements.txt
+    conda env update \
+        -n $CONDA_ENV \
+        --file ./env.yml || exit -1
+    conda install \
+        -q \
+        -y \
+        -n $CONDA_ENV \
+            doxygen \
+            rstcheck || exit -1
     # check reStructuredText formatting
     cd $BUILD_DIRECTORY/python-package
     rstcheck --report warning $(find . -type f -name "*.rst") || exit -1

.gitignore

@@ -346,6 +346,7 @@ instance/
 # Sphinx documentation
 docs/_build/
 docs/pythonapi/
+*.flag
 
 # Doxygen documentation
 docs/doxyoutput/

.readthedocs.yaml

 version: 2
+build:
+  os: "ubuntu-20.04"
+  tools:
+    python: "miniconda3-4.7"
+conda:
+  environment: docs/env.yml
 formats:
   - pdf
-python:
-  version: 3
-  install:
-    - requirements: docs/requirements.txt
 sphinx:
   builder: html
   configuration: docs/conf.py

docs/README.rst

@@ -13,20 +13,55 @@ After each commit on ``master``, documentation is updated and published to `Read
 Build
 -----
 
-You can build the documentation locally. Just install Doxygen and run in ``docs`` folder
+It is not necessary to re-build this documentation while modifying LightGBM's source code.
+The HTML files generated using ``Sphinx`` are not checked into source control.
+However, you may want to build them locally during development to test changes.
+
+Docker
+^^^^^^
+
+The most reliable way to build the documentation locally is with Docker, using `the same images Read the Docs uses <https://hub.docker.com/r/readthedocs/build>`_.
+
+Run the following from the root of this repository to pull the relevant image and run a container locally.
+
+.. code:: sh
+
+    docker run \
+        --rm \
+        --user=0 \
+        -v $(pwd):/opt/LightGBM \
+        --env C_API=true \
+        --env CONDA=/opt/miniforge \
+        --env READTHEDOCS=true \
+        --workdir=/opt/LightGBM/docs \
+        --entrypoint="" \
+        -it readthedocs/build:ubuntu-20.04-2021.09.23 \
+        /bin/bash build-docs.sh
+
+When that code completes, open ``docs/_build/html/index.html`` in your browser.
+
+.. note::
+
+    The navigation in these locally-built docs does not link to the local copy of the R documentation. To view the local version of the R docs, open ``docs/_build/html/R/index.html`` in your browser.
+
+Without Docker
+^^^^^^^^^^^^^^
+
+You can build the documentation locally without Docker. Just install Doxygen and run in ``docs`` folder
 
 .. code:: sh
 
-    pip install -r requirements.txt
+    pip install breathe sphinx 'sphinx_rtd_theme>=0.5'
     make html
 
-Unfortunately, documentation for R code is built only on our site, and commands above will not build it for you locally.
+Note that this will not build the R documentation.
 Consider using common R utilities for documentation generation, if you need it.
+Or use the Docker-based approach described above to build the R documentation locally.
 
 If you faced any problems with Doxygen installation or you simply do not need documentation for C code, it is possible to build the documentation without it:
 
 .. code:: sh
 
-    pip install -r requirements_base.txt
+    pip install sphinx 'sphinx_rtd_theme>=0.5'
     export C_API=NO || set C_API=NO
     make html

docs/build-docs.sh

+#!/bin/bash
+
+rm -f ./_FIRST_RUN.flag
+
+export PATH="${CONDA}/bin:${PATH}"
+
+curl \
+    -sL \
+    -o ${HOME}/miniforge.sh \
+    https://github.com/conda-forge/miniforge/releases/latest/download/Miniforge3-Linux-x86_64.sh
+
+/bin/bash ${HOME}/miniforge.sh -b -p ${CONDA}
+
+conda config --set always_yes yes --set changeps1 no
+conda update -q -y conda
+
+conda env create \
+    --name docs-env \
+    --file env.yml || exit -1
+
+source activate docs-env
+make clean html || exit -1
+
+echo "Done building docs. Open docs/_build/html/index.html in a web browser to view them."

docs/conf.py

@@ -259,21 +259,6 @@ def generate_r_docs(app: Sphinx) -> None:
         The application object representing the Sphinx process.
     """
     commands = f"""
-    /home/docs/.conda/bin/conda create \
-        -q \
-        -y \
-        -c conda-forge \
-        --override-channels \
-        -n r_env \
-            r-base=4.1.0=hb67fd72_2 \
-            r-data.table=1.14.0=r41hcfec24a_0 \
-            r-jsonlite=1.7.2=r41hcfec24a_0 \
-            r-knitr=1.35=r41hc72bb7e_0 \
-            r-matrix=1.3_4=r41he454529_0 \
-            r-pkgdown=1.6.1=r41hc72bb7e_0 \
-            r-rmarkdown=2.11=r41hc72bb7e_0 \
-            r-roxygen2=7.1.1=r41h03ef668_0
-    source /home/docs/.conda/bin/activate r_env
     export TAR=/bin/tar
     cd {CURR_PATH.parent}
     export R_LIBS="$CONDA_PREFIX/lib/R/library"
@@ -298,6 +283,7 @@ def generate_r_docs(app: Sphinx) -> None:
     cd {CURR_PATH.parent}
     """
     try:
+        print("Building R-package documentation")
         # Warning! The following code can cause buffer overflows on RTD.
         # Consider suppressing output completely if RTD project silently fails.
         # Refer to https://github.com/svenevs/exhale
@@ -311,6 +297,7 @@ def generate_r_docs(app: Sphinx) -> None:
             raise RuntimeError(output)
         else:
             print(output)
+            print("Done building R-package documentation")
     except BaseException as e:
         raise Exception(f"An error has occurred while generating documentation for R-package\n{e}")
 

docs/env.yml

+name: docs-env
+channels:
+  - nodefaults
+  - conda-forge
+dependencies:
+  - breathe
+  - python=3.9
+  - r-base=4.1.2
+  - r-data.table=1.14.2
+  - r-jsonlite=1.7.2
+  - r-knitr=1.37
+  - r-matrix=1.4_0
+  - r-pkgdown=1.6.1
+  - r-rmarkdown=2.11
+  - r-roxygen2=7.1.2
+  - sphinx
+  - "sphinx_rtd_theme>=0.5"

docs/requirements.txt

--r requirements_base.txt
-breathe

docs/requirements_base.txt

-sphinx
-sphinx_rtd_theme >= 0.5