[ci] update to 1ES image pool (#6866)

Co-authored-by: Nikita Titov <nekit94-08@mail.ru>
Co-authored-by: Yu Shi <yushi2@microsoft.com>
Co-authored-by: James Lamb <jaylamb20@gmail.com>

.vsts-ci.yml

@@ -27,14 +27,14 @@ resources:
   # ref: https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/resources-containers-container
   containers:
     - container: linux-artifact-builder
-      image: lightgbm/vsts-agent:manylinux_2_28_x86_64
+      image: lightgbm.azurecr.io/vsts-agent:manylinux_2_28_x86_64
       mountReadOnly:
         work: false
         externals: true
         tools: true
         tasks: true
     - container: ubuntu-latest
-      image: 'ubuntu:22.04'
+      image: mcr.microsoft.com/mirror/docker/library/ubuntu:22.04
       options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
       mountReadOnly:
         work: false
@@ -49,181 +49,180 @@ resources:
         tools: true
         tasks: true
 jobs:
-  # TODO: add these jobs back when issues from https://github.com/microsoft/LightGBM/issues/6918 are resolved
-  # ###############
-  # # Maintenance #
-  # ###############
-  # - job: Maintenance
-  #   pool: mariner-20240410-0
-  #   container: ubuntu-latest
-  #   # routine maintenance (like periodically deleting old files),
-  #   # to be run on 1 random CI runner in the self-hosted pool each runner
-  #   steps:
-  #     - script: |
-  #         print-diagnostics(){
-  #           echo "---- df -h -m ----"
-  #           df -h -m
-  #           echo "---- docker system df ----"
-  #           /tmp/docker system df
-  #           echo "---- docker images ----"
-  #           /tmp/docker images
-  #         }
-  #         # check disk usage
-  #         print-diagnostics
-  #         # remove old containers, container images, volumes
-  #         # ref: https://stackoverflow.com/a/32723127/3986677
-  #         # ref: https://depot.dev/blog/docker-clear-cache#removing-everything-with-docker-system-prune
-  #         echo "---- running 'docker system prune' ----"
-  #         /tmp/docker system prune \
-  #           --all \
-  #           --force \
-  #           --volumes \
-  #           --filter until=720h
-  #         # check disk usage again
-  #         print-diagnostics
-  #       displayName: Clean
-  # #########
-  # # Linux #
-  # #########
-  # - job: Linux
-  #   variables:
-  #     COMPILER: gcc
-  #     SETUP_CONDA: 'false'
-  #     OS_NAME: 'linux'
-  #     PRODUCES_ARTIFACTS: 'true'
-  #   pool: mariner-20240410-0
-  #   container: linux-artifact-builder
-  #   strategy:
-  #     matrix:
-  #       regular:
-  #         TASK: regular
-  #         PYTHON_VERSION: '3.11'
-  #       sdist:
-  #         TASK: sdist
-  #         PYTHON_VERSION: '3.9'
-  #       bdist:
-  #         TASK: bdist
-  #         PYTHON_VERSION: '3.10'
-  #       inference:
-  #         TASK: if-else
-  #       mpi_source:
-  #         TASK: mpi
-  #         METHOD: source
-  #         PYTHON_VERSION: '3.10'
-  #       gpu_source:
-  #         TASK: gpu
-  #         METHOD: source
-  #       swig:
-  #         TASK: swig
-  #   steps:
-  #     - script: |
-  #         echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
-  #         echo "##vso[task.prependpath]/usr/lib64/openmpi/bin"
-  #         echo "##vso[task.prependpath]$CONDA/bin"
-  #       displayName: 'Set variables'
-  #     - script: |
-  #         git clean -d -f -x
-  #       displayName: 'Clean source directory'
-  #     - script: |
-  #         echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt'
-  #       displayName: 'Add commit hash to artifacts archive'
-  #     - task: Bash@3
-  #       displayName: Setup
-  #       inputs:
-  #         filePath: $(Build.SourcesDirectory)/.ci/setup.sh
-  #         targetType: filePath
-  #     - task: Bash@3
-  #       displayName: Test
-  #       inputs:
-  #         filePath: $(Build.SourcesDirectory)/.ci/test.sh
-  #         targetType: filePath
-  #     - task: PublishBuildArtifacts@1
-  #       condition: >
-  #         and(
-  #         succeeded(),
-  #         in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'),
-  #         not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
-  #         )
-  #       inputs:
-  #         pathtoPublish: '$(Build.ArtifactStagingDirectory)'
-  #         artifactName: PackageAssets
-  #         artifactType: container
-  # ################
-  # # Linux_latest #
-  # ################
-  # - job: Linux_latest
-  #   variables:
-  #     COMPILER: clang-17
-  #     DEBIAN_FRONTEND: 'noninteractive'
-  #     IN_UBUNTU_BASE_CONTAINER: 'true'
-  #     OS_NAME: 'linux'
-  #     SETUP_CONDA: 'true'
-  #   pool: mariner-20240410-0
-  #   container: ubuntu-latest
-  #   strategy:
-  #     matrix:
-  #       regular:
-  #         TASK: regular
-  #       sdist:
-  #         TASK: sdist
-  #       bdist:
-  #         TASK: bdist
-  #         PYTHON_VERSION: '3.11'
-  #       inference:
-  #         TASK: if-else
-  #       mpi_source:
-  #         TASK: mpi
-  #         METHOD: source
-  #       mpi_pip:
-  #         TASK: mpi
-  #         METHOD: pip
-  #         PYTHON_VERSION: '3.12'
-  #       mpi_wheel:
-  #         TASK: mpi
-  #         METHOD: wheel
-  #         PYTHON_VERSION: '3.10'
-  #       gpu_source:
-  #         TASK: gpu
-  #         METHOD: source
-  #         PYTHON_VERSION: '3.12'
-  #       gpu_pip:
-  #         TASK: gpu
-  #         METHOD: pip
-  #         PYTHON_VERSION: '3.11'
-  #       gpu_wheel:
-  #         TASK: gpu
-  #         METHOD: wheel
-  #         PYTHON_VERSION: '3.10'
-  #       cpp_tests:
-  #         TASK: cpp-tests
-  #         METHOD: with-sanitizers
-  #   steps:
-  #     - script: |
-  #         echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
-  #         CONDA=$HOME/miniforge
-  #         echo "##vso[task.setvariable variable=CONDA]$CONDA"
-  #         echo "##vso[task.prependpath]$CONDA/bin"
-  #       displayName: 'Set variables'
-  #     # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
-  #     - script: |
-  #         /tmp/docker exec -t -u 0 ci-container \
-  #         sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
-  #       displayName: 'Install sudo'
-  #     - script: |
-  #         sudo apt-get update
-  #         sudo apt-get install -y --no-install-recommends git
-  #         git clean -d -f -x
-  #       displayName: 'Clean source directory'
-  #     - task: Bash@3
-  #       displayName: Setup
-  #       inputs:
-  #         filePath: $(Build.SourcesDirectory)/.ci/setup.sh
-  #         targetType: 'filePath'
-  #     - task: Bash@3
-  #       displayName: Test
-  #       inputs:
-  #         filePath: $(Build.SourcesDirectory)/.ci/test.sh
-  #         targetType: 'filePath'
+  ###############
+  # Maintenance #
+  ###############
+  - job: Maintenance
+    pool: lgb_agent_pool_ado
+    container: ubuntu-latest
+    # routine maintenance (like periodically deleting old files),
+    # to be run on 1 random CI runner in the self-hosted pool each runner
+    steps:
+      - script: |
+          print-diagnostics(){
+            echo "---- df -h -m ----"
+            df -h -m
+            echo "---- docker system df ----"
+            /tmp/docker system df
+            echo "---- docker images ----"
+            /tmp/docker images
+          }
+          # check disk usage
+          print-diagnostics
+          # remove old containers, container images, volumes
+          # ref: https://stackoverflow.com/a/32723127/3986677
+          # ref: https://depot.dev/blog/docker-clear-cache#removing-everything-with-docker-system-prune
+          echo "---- running 'docker system prune' ----"
+          /tmp/docker system prune \
+            --all \
+            --force \
+            --volumes \
+            --filter until=720h
+          # check disk usage again
+          print-diagnostics
+        displayName: Clean
+  #########
+  # Linux #
+  #########
+  - job: Linux
+    variables:
+      COMPILER: gcc
+      SETUP_CONDA: 'false'
+      OS_NAME: 'linux'
+      PRODUCES_ARTIFACTS: 'true'
+    pool: lgb_agent_pool_ado
+    container: linux-artifact-builder
+    strategy:
+      matrix:
+        regular:
+          TASK: regular
+          PYTHON_VERSION: '3.11'
+        sdist:
+          TASK: sdist
+          PYTHON_VERSION: '3.9'
+        bdist:
+          TASK: bdist
+          PYTHON_VERSION: '3.10'
+        inference:
+          TASK: if-else
+        mpi_source:
+          TASK: mpi
+          METHOD: source
+          PYTHON_VERSION: '3.10'
+        gpu_source:
+          TASK: gpu
+          METHOD: source
+        swig:
+          TASK: swig
+    steps:
+      - script: |
+          echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
+          echo "##vso[task.prependpath]/usr/lib64/openmpi/bin"
+          echo "##vso[task.prependpath]$CONDA/bin"
+        displayName: 'Set variables'
+      - script: |
+          git clean -d -f -x
+        displayName: 'Clean source directory'
+      - script: |
+          echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt'
+        displayName: 'Add commit hash to artifacts archive'
+      - task: Bash@3
+        displayName: Setup
+        inputs:
+          filePath: $(Build.SourcesDirectory)/.ci/setup.sh
+          targetType: filePath
+      - task: Bash@3
+        displayName: Test
+        inputs:
+          filePath: $(Build.SourcesDirectory)/.ci/test.sh
+          targetType: filePath
+      - task: PublishBuildArtifacts@1
+        condition: >
+          and(
+          succeeded(),
+          in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'),
+          not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
+          )
+        inputs:
+          pathtoPublish: '$(Build.ArtifactStagingDirectory)'
+          artifactName: PackageAssets
+          artifactType: container
+  ################
+  # Linux_latest #
+  ################
+  - job: Linux_latest
+    variables:
+      COMPILER: clang-17
+      DEBIAN_FRONTEND: 'noninteractive'
+      IN_UBUNTU_BASE_CONTAINER: 'true'
+      OS_NAME: 'linux'
+      SETUP_CONDA: 'true'
+    pool: lgb_agent_pool_ado
+    container: ubuntu-latest
+    strategy:
+      matrix:
+        regular:
+          TASK: regular
+        sdist:
+          TASK: sdist
+        bdist:
+          TASK: bdist
+          PYTHON_VERSION: '3.11'
+        inference:
+          TASK: if-else
+        mpi_source:
+          TASK: mpi
+          METHOD: source
+        mpi_pip:
+          TASK: mpi
+          METHOD: pip
+          PYTHON_VERSION: '3.12'
+        mpi_wheel:
+          TASK: mpi
+          METHOD: wheel
+          PYTHON_VERSION: '3.10'
+        gpu_source:
+          TASK: gpu
+          METHOD: source
+          PYTHON_VERSION: '3.12'
+        gpu_pip:
+          TASK: gpu
+          METHOD: pip
+          PYTHON_VERSION: '3.11'
+        gpu_wheel:
+          TASK: gpu
+          METHOD: wheel
+          PYTHON_VERSION: '3.10'
+        cpp_tests:
+          TASK: cpp-tests
+          METHOD: with-sanitizers
+    steps:
+      - script: |
+          echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
+          CONDA=$HOME/miniforge
+          echo "##vso[task.setvariable variable=CONDA]$CONDA"
+          echo "##vso[task.prependpath]$CONDA/bin"
+        displayName: 'Set variables'
+      # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
+      - script: |
+          /tmp/docker exec -t -u 0 ci-container \
+          sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
+        displayName: 'Install sudo'
+      - script: |
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends git
+          git clean -d -f -x
+        displayName: 'Clean source directory'
+      - task: Bash@3
+        displayName: Setup
+        inputs:
+          filePath: $(Build.SourcesDirectory)/.ci/setup.sh
+          targetType: 'filePath'
+      - task: Bash@3
+        displayName: Test
+        inputs:
+          filePath: $(Build.SourcesDirectory)/.ci/test.sh
+          targetType: 'filePath'
   #########
   # macOS #
   #########
@@ -365,9 +364,8 @@ jobs:
   ###########
   - job: Package
     dependsOn:
-      # TODO: add jobs back when https://github.com/microsoft/LightGBM/issues/6918 is resolved
-      # - Linux
-      # - Linux_latest
+      - Linux
+      - Linux_latest
       - macOS
       - Windows
       - R_artifact
@@ -407,20 +405,19 @@ jobs:
         inputs:
           artifactName: R-package
           downloadPath: $(Build.SourcesDirectory)/R
-      # TODO: add scripts back when https://github.com/microsoft/LightGBM/issues/6918 is resolved
-      # - script: |
-      #     python "$(Build.SourcesDirectory)/.ci/create-nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
-      #   displayName: 'Create NuGet configuration files'
-      # - task: NuGetCommand@2
-      #   inputs:
-      #     command: pack
-      #     packagesToPack: '$(Build.SourcesDirectory)/.ci/nuget/*.nuspec'
-      #     packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
-      # - task: PublishBuildArtifacts@1
-      #   inputs:
-      #     pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
-      #     artifactName: NuGet
-      #     artifactType: container
+      - script: |
+          python "$(Build.SourcesDirectory)/.ci/create-nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
+        displayName: 'Create NuGet configuration files'
+      - task: NuGetCommand@2
+        inputs:
+          command: pack
+          packagesToPack: '$(Build.SourcesDirectory)/.ci/nuget/*.nuspec'
+          packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
+      - task: PublishBuildArtifacts@1
+        inputs:
+          pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
+          artifactName: NuGet
+          artifactType: container
       - task: GitHubRelease@0
         displayName: 'Create GitHub Release'
         condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
@@ -431,11 +428,10 @@ jobs:
           target: '$(Build.SourceVersion)'
           tagSource: 'auto'
           title: '$(Build.SourceBranchName)'
-          # TODO: add '$(Build.ArtifactStagingDirectory)/nuget/*.nupkg' artifact
-          # back when https://github.com/microsoft/LightGBM/issues/6918 is resolved
           assets: |
             $(Build.SourcesDirectory)/binaries/PackageAssets/*
             $(Build.SourcesDirectory)/R/R-package/*
+            $(Build.ArtifactStagingDirectory)/nuget/*.nupkg
             $(Build.ArtifactStagingDirectory)/archives/*
           assetUploadMode: 'delete'
           isDraft: true