.vsts-ci.yml

trigger:
  branches:
    include:
      - master
  tags:
    include:
      - v*
pr:
  - master
variables:
  AZURE: 'true'
  CMAKE_BUILD_PARALLEL_LEVEL: 4
  PYTHON_VERSION: '3.13'
  runCodesignValidationInjection: false
  skipComponentGovernanceDetection: true
  Codeql.Enabled: false
  Codeql.SkipTaskAutoInjection: true
  DOTNET_CLI_TELEMETRY_OPTOUT: true
  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
  SKBUILD_STRICT_CONFIG: true
resources:
  # The __work/ directory, where Azure DevOps writes the source files, needs to be read-write because
  # LightGBM's CI jobs write files in the source directory.
  #
  # For all the containers included here, all other directories that Azure mounts in are mounted as read-only
  # to minimize the risk of side effects from one run affecting future runs.
  # ref: https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/resources-containers-container
  containers:
    - container: linux-artifact-builder
      image: lightgbm/vsts-agent:manylinux_2_28_x86_64
      mountReadOnly:
        work: false
        externals: true
        tools: true
        tasks: true
    - container: ubuntu-latest
      image: 'ubuntu:22.04'
      options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
      mountReadOnly:
        work: false
        externals: true
        tools: true
        tasks: true
    - container: rbase
      image: wch1/r-debug
      mountReadOnly:
        work: false
        externals: true
        tools: true
        tasks: true
jobs:
  # TODO: add these jobs back when issues from https://github.com/microsoft/LightGBM/issues/6918 are resolved
  # ###############
  # # Maintenance #
  # ###############
  # - job: Maintenance
  #   pool: mariner-20240410-0
  #   container: ubuntu-latest
  #   # routine maintenance (like periodically deleting old files),
  #   # to be run on 1 random CI runner in the self-hosted pool each runner
  #   steps:
  #     - script: |
  #         print-diagnostics(){
  #           echo "---- df -h -m ----"
  #           df -h -m
  #           echo "---- docker system df ----"
  #           /tmp/docker system df
  #           echo "---- docker images ----"
  #           /tmp/docker images
  #         }
  #         # check disk usage
  #         print-diagnostics
  #         # remove old containers, container images, volumes
  #         # ref: https://stackoverflow.com/a/32723127/3986677
  #         # ref: https://depot.dev/blog/docker-clear-cache#removing-everything-with-docker-system-prune
  #         echo "---- running 'docker system prune' ----"
  #         /tmp/docker system prune \
  #           --all \
  #           --force \
  #           --volumes \
  #           --filter until=720h
  #         # check disk usage again
  #         print-diagnostics
  #       displayName: Clean
  # #########
  # # Linux #
  # #########
  # - job: Linux
  #   variables:
  #     COMPILER: gcc
  #     SETUP_CONDA: 'false'
  #     OS_NAME: 'linux'
  #     PRODUCES_ARTIFACTS: 'true'
  #   pool: mariner-20240410-0
  #   container: linux-artifact-builder
  #   strategy:
  #     matrix:
  #       regular:
  #         TASK: regular
  #         PYTHON_VERSION: '3.11'
  #       sdist:
  #         TASK: sdist
  #         PYTHON_VERSION: '3.9'
  #       bdist:
  #         TASK: bdist
  #         PYTHON_VERSION: '3.10'
  #       inference:
  #         TASK: if-else
  #       mpi_source:
  #         TASK: mpi
  #         METHOD: source
  #         PYTHON_VERSION: '3.10'
  #       gpu_source:
  #         TASK: gpu
  #         METHOD: source
  #       swig:
  #         TASK: swig
  #   steps:
  #     - script: |
  #         echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
  #         echo "##vso[task.prependpath]/usr/lib64/openmpi/bin"
  #         echo "##vso[task.prependpath]$CONDA/bin"
  #       displayName: 'Set variables'
  #     - script: |
  #         git clean -d -f -x
  #       displayName: 'Clean source directory'
  #     - script: |
  #         echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt'
  #       displayName: 'Add commit hash to artifacts archive'
  #     - task: Bash@3
  #       displayName: Setup
  #       inputs:
  #         filePath: $(Build.SourcesDirectory)/.ci/setup.sh
  #         targetType: filePath
  #     - task: Bash@3
  #       displayName: Test
  #       inputs:
  #         filePath: $(Build.SourcesDirectory)/.ci/test.sh
  #         targetType: filePath
  #     - task: PublishBuildArtifacts@1
  #       condition: >
  #         and(
  #         succeeded(),
  #         in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'),
  #         not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
  #         )
  #       inputs:
  #         pathtoPublish: '$(Build.ArtifactStagingDirectory)'
  #         artifactName: PackageAssets
  #         artifactType: container
  # ################
  # # Linux_latest #
  # ################
  # - job: Linux_latest
  #   variables:
  #     COMPILER: clang-17
  #     DEBIAN_FRONTEND: 'noninteractive'
  #     IN_UBUNTU_BASE_CONTAINER: 'true'
  #     OS_NAME: 'linux'
  #     SETUP_CONDA: 'true'
  #   pool: mariner-20240410-0
  #   container: ubuntu-latest
  #   strategy:
  #     matrix:
  #       regular:
  #         TASK: regular
  #       sdist:
  #         TASK: sdist
  #       bdist:
  #         TASK: bdist
  #         PYTHON_VERSION: '3.11'
  #       inference:
  #         TASK: if-else
  #       mpi_source:
  #         TASK: mpi
  #         METHOD: source
  #       mpi_pip:
  #         TASK: mpi
  #         METHOD: pip
  #         PYTHON_VERSION: '3.12'
  #       mpi_wheel:
  #         TASK: mpi
  #         METHOD: wheel
  #         PYTHON_VERSION: '3.10'
  #       gpu_source:
  #         TASK: gpu
  #         METHOD: source
  #         PYTHON_VERSION: '3.12'
  #       gpu_pip:
  #         TASK: gpu
  #         METHOD: pip
  #         PYTHON_VERSION: '3.11'
  #       gpu_wheel:
  #         TASK: gpu
  #         METHOD: wheel
  #         PYTHON_VERSION: '3.10'
  #       cpp_tests:
  #         TASK: cpp-tests
  #         METHOD: with-sanitizers
  #   steps:
  #     - script: |
  #         echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
  #         CONDA=$HOME/miniforge
  #         echo "##vso[task.setvariable variable=CONDA]$CONDA"
  #         echo "##vso[task.prependpath]$CONDA/bin"
  #       displayName: 'Set variables'
  #     # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
  #     - script: |
  #         /tmp/docker exec -t -u 0 ci-container \
  #         sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
  #       displayName: 'Install sudo'
  #     - script: |
  #         sudo apt-get update
  #         sudo apt-get install -y --no-install-recommends git
  #         git clean -d -f -x
  #       displayName: 'Clean source directory'
  #     - task: Bash@3
  #       displayName: Setup
  #       inputs:
  #         filePath: $(Build.SourcesDirectory)/.ci/setup.sh
  #         targetType: 'filePath'
  #     - task: Bash@3
  #       displayName: Test
  #       inputs:
  #         filePath: $(Build.SourcesDirectory)/.ci/test.sh
  #         targetType: 'filePath'
  #########
  # macOS #
  #########
  - job: macOS
    variables:
      COMPILER: clang
      OS_NAME: 'macos'
      PRODUCES_ARTIFACTS: 'true'
    pool:
      vmImage: 'macOS-13'
    strategy:
      matrix:
        regular:
          TASK: regular
          PYTHON_VERSION: '3.11'
        sdist:
          TASK: sdist
          PYTHON_VERSION: '3.10'
        bdist:
          TASK: bdist
        swig:
          TASK: swig
        cpp_tests:
          TASK: cpp-tests
          METHOD: with-sanitizers
          SANITIZERS: "address;undefined"
    steps:
      - script: |
          echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
          CONDA=$AGENT_HOMEDIRECTORY/miniforge
          echo "##vso[task.setvariable variable=CONDA]$CONDA"
          echo "##vso[task.prependpath]$CONDA/bin"
          echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64"
        displayName: 'Set variables'
      - script: |
          git clean -d -f -x
        displayName: 'Clean source directory'
      - task: Bash@3
        displayName: Setup
        inputs:
          filePath: $(Build.SourcesDirectory)/.ci/setup.sh
          targetType: filePath
      - task: Bash@3
        displayName: Test
        inputs:
          filePath: $(Build.SourcesDirectory)/.ci/test.sh
          targetType: filePath
      - task: PublishBuildArtifacts@1
        condition: >
          and(
          succeeded(),
          in(variables['TASK'], 'regular', 'bdist', 'swig'),
          not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
          )
        inputs:
          pathtoPublish: '$(Build.ArtifactStagingDirectory)'
          artifactName: PackageAssets
          artifactType: container
  ###########
  # Windows #
  ###########
  - job: Windows
    pool:
      vmImage: 'windows-2019'
    strategy:
      matrix:
        regular:
          TASK: regular
          PYTHON_VERSION: '3.11'
        sdist:
          TASK: sdist
          PYTHON_VERSION: '3.10'
        bdist:
          TASK: bdist
        swig:
          TASK: swig
        cpp_tests:
          TASK: cpp-tests
    steps:
      - powershell: |
          Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
        displayName: 'Set Variables'
      - script: |
          git clean -d -f -x
        displayName: 'Clean source directory'
      - script: |
          cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install-opencl.ps1"
        condition: eq(variables['TASK'], 'bdist')
        displayName: 'Install OpenCL'
      - script: |
          cmd /c "conda config --remove channels defaults"
          cmd /c "conda config --add channels nodefaults"
          cmd /c "conda config --add channels conda-forge"
          cmd /c "conda config --set channel_priority strict"
          cmd /c "conda init powershell"
          cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test-windows.ps1"
        displayName: Test
      - task: PublishBuildArtifacts@1
        condition: >
          and(
          succeeded(),
          in(variables['TASK'], 'regular', 'bdist', 'swig'),
          not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
          )
        inputs:
          pathtoPublish: '$(Build.ArtifactStagingDirectory)'
          artifactName: PackageAssets
          artifactType: container
  ##############
  # R_artifact #
  ##############
  - job: R_artifact
    condition: not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
    pool:
      vmImage: 'ubuntu-22.04'
    container: rbase
    steps:
      - script: |
          git clean -d -f -x
        displayName: 'Clean source directory'
      - script: |
          LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g")
          R_LIB_PATH=~/Rlib
          export R_LIBS=${R_LIB_PATH}
          mkdir -p ${R_LIB_PATH}
          RDscript .ci/install-r-deps.R --build --include=RhpcBLASctl || exit 1
          sh build-cran-package.sh --r-executable=RD || exit 1
          mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz
        displayName: 'Build CRAN R-package'
      - task: PublishBuildArtifacts@1
        condition: succeeded()
        inputs:
          pathtoPublish: $(Build.ArtifactStagingDirectory)
          artifactName: R-package
          artifactType: container

  ###########
  # Package #
  ###########
  - job: Package
    dependsOn:
      # TODO: add jobs back when https://github.com/microsoft/LightGBM/issues/6918 is resolved
      # - Linux
      # - Linux_latest
      - macOS
      - Windows
      - R_artifact
    condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
    pool:
      vmImage: 'ubuntu-22.04'
    steps:
      # Create archives with complete source code included (with git submodules)
      - task: ArchiveFiles@2
        displayName: Create zip archive
        condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)
          includeRootFolder: false
          archiveType: zip
          archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_zip.zip'
          replaceExistingArchive: true
      - task: ArchiveFiles@2
        displayName: Create tar.gz archive
        condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
        inputs:
          rootFolderOrFile: $(Build.SourcesDirectory)
          includeRootFolder: false
          archiveType: tar
          tarCompression: gz
          archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_tar_gz.tar.gz'
          replaceExistingArchive: true
      # Download all agent packages from all previous phases
      - task: DownloadBuildArtifacts@0
        displayName: Download package assets
        inputs:
          artifactName: PackageAssets
          downloadPath: $(Build.SourcesDirectory)/binaries
      - task: DownloadBuildArtifacts@0
        displayName: Download R-package
        condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
        inputs:
          artifactName: R-package
          downloadPath: $(Build.SourcesDirectory)/R
      # TODO: add scripts back when https://github.com/microsoft/LightGBM/issues/6918 is resolved
      # - script: |
      #     python "$(Build.SourcesDirectory)/.ci/create-nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
      #   displayName: 'Create NuGet configuration files'
      # - task: NuGetCommand@2
      #   inputs:
      #     command: pack
      #     packagesToPack: '$(Build.SourcesDirectory)/.ci/nuget/*.nuspec'
      #     packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
      # - task: PublishBuildArtifacts@1
      #   inputs:
      #     pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
      #     artifactName: NuGet
      #     artifactType: container
      - task: GitHubRelease@0
        displayName: 'Create GitHub Release'
        condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
        inputs:
          gitHubConnection: guolinke
          repositoryName: '$(Build.Repository.Name)'
          action: 'create'
          target: '$(Build.SourceVersion)'
          tagSource: 'auto'
          title: '$(Build.SourceBranchName)'
          # TODO: add '$(Build.ArtifactStagingDirectory)/nuget/*.nupkg' artifact
          # back when https://github.com/microsoft/LightGBM/issues/6918 is resolved
          assets: |
            $(Build.SourcesDirectory)/binaries/PackageAssets/*
            $(Build.SourcesDirectory)/R/R-package/*
            $(Build.ArtifactStagingDirectory)/archives/*
          assetUploadMode: 'delete'
          isDraft: true
          isPreRelease: false
          addChangeLog: false