Unverified Commit a79e0332 authored by shiyu1994's avatar shiyu1994 Committed by GitHub
Browse files

[ci] update to 1ES image pool (#6866)


Co-authored-by: default avatarNikita Titov <nekit94-08@mail.ru>
Co-authored-by: default avatarYu Shi <yushi2@microsoft.com>
Co-authored-by: default avatarJames Lamb <jaylamb20@gmail.com>
parent 72a39817
......@@ -27,14 +27,14 @@ resources:
# ref: https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/resources-containers-container
containers:
- container: linux-artifact-builder
image: lightgbm/vsts-agent:manylinux_2_28_x86_64
image: lightgbm.azurecr.io/vsts-agent:manylinux_2_28_x86_64
mountReadOnly:
work: false
externals: true
tools: true
tasks: true
- container: ubuntu-latest
image: 'ubuntu:22.04'
image: mcr.microsoft.com/mirror/docker/library/ubuntu:22.04
options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
mountReadOnly:
work: false
......@@ -49,181 +49,180 @@ resources:
tools: true
tasks: true
jobs:
# TODO: add these jobs back when issues from https://github.com/microsoft/LightGBM/issues/6918 are resolved
# ###############
# # Maintenance #
# ###############
# - job: Maintenance
# pool: mariner-20240410-0
# container: ubuntu-latest
# # routine maintenance (like periodically deleting old files),
# # to be run on 1 random CI runner in the self-hosted pool each runner
# steps:
# - script: |
# print-diagnostics(){
# echo "---- df -h -m ----"
# df -h -m
# echo "---- docker system df ----"
# /tmp/docker system df
# echo "---- docker images ----"
# /tmp/docker images
# }
# # check disk usage
# print-diagnostics
# # remove old containers, container images, volumes
# # ref: https://stackoverflow.com/a/32723127/3986677
# # ref: https://depot.dev/blog/docker-clear-cache#removing-everything-with-docker-system-prune
# echo "---- running 'docker system prune' ----"
# /tmp/docker system prune \
# --all \
# --force \
# --volumes \
# --filter until=720h
# # check disk usage again
# print-diagnostics
# displayName: Clean
# #########
# # Linux #
# #########
# - job: Linux
# variables:
# COMPILER: gcc
# SETUP_CONDA: 'false'
# OS_NAME: 'linux'
# PRODUCES_ARTIFACTS: 'true'
# pool: mariner-20240410-0
# container: linux-artifact-builder
# strategy:
# matrix:
# regular:
# TASK: regular
# PYTHON_VERSION: '3.11'
# sdist:
# TASK: sdist
# PYTHON_VERSION: '3.9'
# bdist:
# TASK: bdist
# PYTHON_VERSION: '3.10'
# inference:
# TASK: if-else
# mpi_source:
# TASK: mpi
# METHOD: source
# PYTHON_VERSION: '3.10'
# gpu_source:
# TASK: gpu
# METHOD: source
# swig:
# TASK: swig
# steps:
# - script: |
# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
# echo "##vso[task.prependpath]/usr/lib64/openmpi/bin"
# echo "##vso[task.prependpath]$CONDA/bin"
# displayName: 'Set variables'
# - script: |
# git clean -d -f -x
# displayName: 'Clean source directory'
# - script: |
# echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt'
# displayName: 'Add commit hash to artifacts archive'
# - task: Bash@3
# displayName: Setup
# inputs:
# filePath: $(Build.SourcesDirectory)/.ci/setup.sh
# targetType: filePath
# - task: Bash@3
# displayName: Test
# inputs:
# filePath: $(Build.SourcesDirectory)/.ci/test.sh
# targetType: filePath
# - task: PublishBuildArtifacts@1
# condition: >
# and(
# succeeded(),
# in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'),
# not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
# )
# inputs:
# pathtoPublish: '$(Build.ArtifactStagingDirectory)'
# artifactName: PackageAssets
# artifactType: container
# ################
# # Linux_latest #
# ################
# - job: Linux_latest
# variables:
# COMPILER: clang-17
# DEBIAN_FRONTEND: 'noninteractive'
# IN_UBUNTU_BASE_CONTAINER: 'true'
# OS_NAME: 'linux'
# SETUP_CONDA: 'true'
# pool: mariner-20240410-0
# container: ubuntu-latest
# strategy:
# matrix:
# regular:
# TASK: regular
# sdist:
# TASK: sdist
# bdist:
# TASK: bdist
# PYTHON_VERSION: '3.11'
# inference:
# TASK: if-else
# mpi_source:
# TASK: mpi
# METHOD: source
# mpi_pip:
# TASK: mpi
# METHOD: pip
# PYTHON_VERSION: '3.12'
# mpi_wheel:
# TASK: mpi
# METHOD: wheel
# PYTHON_VERSION: '3.10'
# gpu_source:
# TASK: gpu
# METHOD: source
# PYTHON_VERSION: '3.12'
# gpu_pip:
# TASK: gpu
# METHOD: pip
# PYTHON_VERSION: '3.11'
# gpu_wheel:
# TASK: gpu
# METHOD: wheel
# PYTHON_VERSION: '3.10'
# cpp_tests:
# TASK: cpp-tests
# METHOD: with-sanitizers
# steps:
# - script: |
# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
# CONDA=$HOME/miniforge
# echo "##vso[task.setvariable variable=CONDA]$CONDA"
# echo "##vso[task.prependpath]$CONDA/bin"
# displayName: 'Set variables'
# # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
# - script: |
# /tmp/docker exec -t -u 0 ci-container \
# sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
# displayName: 'Install sudo'
# - script: |
# sudo apt-get update
# sudo apt-get install -y --no-install-recommends git
# git clean -d -f -x
# displayName: 'Clean source directory'
# - task: Bash@3
# displayName: Setup
# inputs:
# filePath: $(Build.SourcesDirectory)/.ci/setup.sh
# targetType: 'filePath'
# - task: Bash@3
# displayName: Test
# inputs:
# filePath: $(Build.SourcesDirectory)/.ci/test.sh
# targetType: 'filePath'
###############
# Maintenance #
###############
- job: Maintenance
pool: lgb_agent_pool_ado
container: ubuntu-latest
# routine maintenance (like periodically deleting old files),
# to be run on 1 random CI runner in the self-hosted pool each runner
steps:
- script: |
print-diagnostics(){
echo "---- df -h -m ----"
df -h -m
echo "---- docker system df ----"
/tmp/docker system df
echo "---- docker images ----"
/tmp/docker images
}
# check disk usage
print-diagnostics
# remove old containers, container images, volumes
# ref: https://stackoverflow.com/a/32723127/3986677
# ref: https://depot.dev/blog/docker-clear-cache#removing-everything-with-docker-system-prune
echo "---- running 'docker system prune' ----"
/tmp/docker system prune \
--all \
--force \
--volumes \
--filter until=720h
# check disk usage again
print-diagnostics
displayName: Clean
#########
# Linux #
#########
- job: Linux
variables:
COMPILER: gcc
SETUP_CONDA: 'false'
OS_NAME: 'linux'
PRODUCES_ARTIFACTS: 'true'
pool: lgb_agent_pool_ado
container: linux-artifact-builder
strategy:
matrix:
regular:
TASK: regular
PYTHON_VERSION: '3.11'
sdist:
TASK: sdist
PYTHON_VERSION: '3.9'
bdist:
TASK: bdist
PYTHON_VERSION: '3.10'
inference:
TASK: if-else
mpi_source:
TASK: mpi
METHOD: source
PYTHON_VERSION: '3.10'
gpu_source:
TASK: gpu
METHOD: source
swig:
TASK: swig
steps:
- script: |
echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
echo "##vso[task.prependpath]/usr/lib64/openmpi/bin"
echo "##vso[task.prependpath]$CONDA/bin"
displayName: 'Set variables'
- script: |
git clean -d -f -x
displayName: 'Clean source directory'
- script: |
echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt'
displayName: 'Add commit hash to artifacts archive'
- task: Bash@3
displayName: Setup
inputs:
filePath: $(Build.SourcesDirectory)/.ci/setup.sh
targetType: filePath
- task: Bash@3
displayName: Test
inputs:
filePath: $(Build.SourcesDirectory)/.ci/test.sh
targetType: filePath
- task: PublishBuildArtifacts@1
condition: >
and(
succeeded(),
in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'),
not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
)
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)'
artifactName: PackageAssets
artifactType: container
################
# Linux_latest #
################
- job: Linux_latest
variables:
COMPILER: clang-17
DEBIAN_FRONTEND: 'noninteractive'
IN_UBUNTU_BASE_CONTAINER: 'true'
OS_NAME: 'linux'
SETUP_CONDA: 'true'
pool: lgb_agent_pool_ado
container: ubuntu-latest
strategy:
matrix:
regular:
TASK: regular
sdist:
TASK: sdist
bdist:
TASK: bdist
PYTHON_VERSION: '3.11'
inference:
TASK: if-else
mpi_source:
TASK: mpi
METHOD: source
mpi_pip:
TASK: mpi
METHOD: pip
PYTHON_VERSION: '3.12'
mpi_wheel:
TASK: mpi
METHOD: wheel
PYTHON_VERSION: '3.10'
gpu_source:
TASK: gpu
METHOD: source
PYTHON_VERSION: '3.12'
gpu_pip:
TASK: gpu
METHOD: pip
PYTHON_VERSION: '3.11'
gpu_wheel:
TASK: gpu
METHOD: wheel
PYTHON_VERSION: '3.10'
cpp_tests:
TASK: cpp-tests
METHOD: with-sanitizers
steps:
- script: |
echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
CONDA=$HOME/miniforge
echo "##vso[task.setvariable variable=CONDA]$CONDA"
echo "##vso[task.prependpath]$CONDA/bin"
displayName: 'Set variables'
# https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
- script: |
/tmp/docker exec -t -u 0 ci-container \
sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
displayName: 'Install sudo'
- script: |
sudo apt-get update
sudo apt-get install -y --no-install-recommends git
git clean -d -f -x
displayName: 'Clean source directory'
- task: Bash@3
displayName: Setup
inputs:
filePath: $(Build.SourcesDirectory)/.ci/setup.sh
targetType: 'filePath'
- task: Bash@3
displayName: Test
inputs:
filePath: $(Build.SourcesDirectory)/.ci/test.sh
targetType: 'filePath'
#########
# macOS #
#########
......@@ -365,9 +364,8 @@ jobs:
###########
- job: Package
dependsOn:
# TODO: add jobs back when https://github.com/microsoft/LightGBM/issues/6918 is resolved
# - Linux
# - Linux_latest
- Linux
- Linux_latest
- macOS
- Windows
- R_artifact
......@@ -407,20 +405,19 @@ jobs:
inputs:
artifactName: R-package
downloadPath: $(Build.SourcesDirectory)/R
# TODO: add scripts back when https://github.com/microsoft/LightGBM/issues/6918 is resolved
# - script: |
# python "$(Build.SourcesDirectory)/.ci/create-nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
# displayName: 'Create NuGet configuration files'
# - task: NuGetCommand@2
# inputs:
# command: pack
# packagesToPack: '$(Build.SourcesDirectory)/.ci/nuget/*.nuspec'
# packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
# - task: PublishBuildArtifacts@1
# inputs:
# pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
# artifactName: NuGet
# artifactType: container
- script: |
python "$(Build.SourcesDirectory)/.ci/create-nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
displayName: 'Create NuGet configuration files'
- task: NuGetCommand@2
inputs:
command: pack
packagesToPack: '$(Build.SourcesDirectory)/.ci/nuget/*.nuspec'
packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
- task: PublishBuildArtifacts@1
inputs:
pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
artifactName: NuGet
artifactType: container
- task: GitHubRelease@0
displayName: 'Create GitHub Release'
condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
......@@ -431,11 +428,10 @@ jobs:
target: '$(Build.SourceVersion)'
tagSource: 'auto'
title: '$(Build.SourceBranchName)'
# TODO: add '$(Build.ArtifactStagingDirectory)/nuget/*.nupkg' artifact
# back when https://github.com/microsoft/LightGBM/issues/6918 is resolved
assets: |
$(Build.SourcesDirectory)/binaries/PackageAssets/*
$(Build.SourcesDirectory)/R/R-package/*
$(Build.ArtifactStagingDirectory)/nuget/*.nupkg
$(Build.ArtifactStagingDirectory)/archives/*
assetUploadMode: 'delete'
isDraft: true
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment