Skip to content

GitLab

"...git@developer.sourcefind.cn:tianlh/lightgbm-dcu.git" did not exist on "e0af160ad49899aea998b222397282f3a13d8226"
Unverified Commit a79e0332 authored by shiyu1994's avatar shiyu1994 Committed by GitHub
Browse files

[ci] update to 1ES image pool (#6866) 


Co-authored-by: default avatarNikita Titov <nekit94-08@mail.ru>
Co-authored-by: default avatarYu Shi <yushi2@microsoft.com>
Co-authored-by: default avatarJames Lamb <jaylamb20@gmail.com>
parent 72a39817
Show whitespace changes
Inline Side-by-side
Showing with 192 additions and 196 deletions
.vsts-ci.yml
View file @ a79e0332
...@@ -27,14 +27,14 @@ resources: ...@@ -27,14 +27,14 @@ resources:
# ref: https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/resources-containers-container # ref: https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/resources-containers-container
containers: containers:
- container: linux-artifact-builder - container: linux-artifact-builder
image: lightgbm/vsts-agent:manylinux_2_28_x86_64 image: lightgbm.azurecr.io/vsts-agent:manylinux_2_28_x86_64
mountReadOnly: mountReadOnly:
work: false work: false
externals: true externals: true
tools: true tools: true
tasks: true tasks: true
- container: ubuntu-latest - container: ubuntu-latest
image: 'ubuntu:22.04' image: mcr.microsoft.com/mirror/docker/library/ubuntu:22.04
options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro" options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
mountReadOnly: mountReadOnly:
work: false work: false
...@@ -49,181 +49,180 @@ resources: ...@@ -49,181 +49,180 @@ resources:
tools: true tools: true
tasks: true tasks: true
jobs: jobs:
# TODO: add these jobs back when issues from https://github.com/microsoft/LightGBM/issues/6918 are resolved ###############
# ############### # Maintenance #
# # Maintenance # ###############
# ############### - job: Maintenance
# - job: Maintenance pool: lgb_agent_pool_ado
# pool: mariner-20240410-0 container: ubuntu-latest
# container: ubuntu-latest # routine maintenance (like periodically deleting old files),
# # routine maintenance (like periodically deleting old files), # to be run on 1 random CI runner in the self-hosted pool each runner
# # to be run on 1 random CI runner in the self-hosted pool each runner steps:
# steps: - script: |
# - script: | print-diagnostics(){
# print-diagnostics(){ echo "---- df -h -m ----"
# echo "---- df -h -m ----" df -h -m
# df -h -m echo "---- docker system df ----"
# echo "---- docker system df ----" /tmp/docker system df
# /tmp/docker system df echo "---- docker images ----"
# echo "---- docker images ----" /tmp/docker images
# /tmp/docker images }
# } # check disk usage
# # check disk usage print-diagnostics
# print-diagnostics # remove old containers, container images, volumes
# # remove old containers, container images, volumes # ref: https://stackoverflow.com/a/32723127/3986677
# # ref: https://stackoverflow.com/a/32723127/3986677 # ref: https://depot.dev/blog/docker-clear-cache#removing-everything-with-docker-system-prune
# # ref: https://depot.dev/blog/docker-clear-cache#removing-everything-with-docker-system-prune echo "---- running 'docker system prune' ----"
# echo "---- running 'docker system prune' ----" /tmp/docker system prune \
# /tmp/docker system prune \ --all \
# --all \ --force \
# --force \ --volumes \
# --volumes \ --filter until=720h
# --filter until=720h # check disk usage again
# # check disk usage again print-diagnostics
# print-diagnostics displayName: Clean
# displayName: Clean #########
# ######### # Linux #
# # Linux # #########
# ######### - job: Linux
# - job: Linux variables:
# variables: COMPILER: gcc
# COMPILER: gcc SETUP_CONDA: 'false'
# SETUP_CONDA: 'false' OS_NAME: 'linux'
# OS_NAME: 'linux' PRODUCES_ARTIFACTS: 'true'
# PRODUCES_ARTIFACTS: 'true' pool: lgb_agent_pool_ado
# pool: mariner-20240410-0 container: linux-artifact-builder
# container: linux-artifact-builder strategy:
# strategy: matrix:
# matrix: regular:
# regular: TASK: regular
# TASK: regular PYTHON_VERSION: '3.11'
# PYTHON_VERSION: '3.11' sdist:
# sdist: TASK: sdist
# TASK: sdist PYTHON_VERSION: '3.9'
# PYTHON_VERSION: '3.9' bdist:
# bdist: TASK: bdist
# TASK: bdist PYTHON_VERSION: '3.10'
# PYTHON_VERSION: '3.10' inference:
# inference: TASK: if-else
# TASK: if-else mpi_source:
# mpi_source: TASK: mpi
# TASK: mpi METHOD: source
# METHOD: source PYTHON_VERSION: '3.10'
# PYTHON_VERSION: '3.10' gpu_source:
# gpu_source: TASK: gpu
# TASK: gpu METHOD: source
# METHOD: source swig:
# swig: TASK: swig
# TASK: swig steps:
# steps: - script: |
# - script: | echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" echo "##vso[task.prependpath]/usr/lib64/openmpi/bin"
# echo "##vso[task.prependpath]/usr/lib64/openmpi/bin" echo "##vso[task.prependpath]$CONDA/bin"
# echo "##vso[task.prependpath]$CONDA/bin" displayName: 'Set variables'
# displayName: 'Set variables' - script: |
# - script: | git clean -d -f -x
# git clean -d -f -x displayName: 'Clean source directory'
# displayName: 'Clean source directory' - script: |
# - script: | echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt'
# echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt' displayName: 'Add commit hash to artifacts archive'
# displayName: 'Add commit hash to artifacts archive' - task: Bash@3
# - task: Bash@3 displayName: Setup
# displayName: Setup inputs:
# inputs: filePath: $(Build.SourcesDirectory)/.ci/setup.sh
# filePath: $(Build.SourcesDirectory)/.ci/setup.sh targetType: filePath
# targetType: filePath - task: Bash@3
# - task: Bash@3 displayName: Test
# displayName: Test inputs:
# inputs: filePath: $(Build.SourcesDirectory)/.ci/test.sh
# filePath: $(Build.SourcesDirectory)/.ci/test.sh targetType: filePath
# targetType: filePath - task: PublishBuildArtifacts@1
# - task: PublishBuildArtifacts@1 condition: >
# condition: > and(
# and( succeeded(),
# succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'),
# in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
# not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')) )
# ) inputs:
# inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)'
# pathtoPublish: '$(Build.ArtifactStagingDirectory)' artifactName: PackageAssets
# artifactName: PackageAssets artifactType: container
# artifactType: container ################
# ################ # Linux_latest #
# # Linux_latest # ################
# ################ - job: Linux_latest
# - job: Linux_latest variables:
# variables: COMPILER: clang-17
# COMPILER: clang-17 DEBIAN_FRONTEND: 'noninteractive'
# DEBIAN_FRONTEND: 'noninteractive' IN_UBUNTU_BASE_CONTAINER: 'true'
# IN_UBUNTU_BASE_CONTAINER: 'true' OS_NAME: 'linux'
# OS_NAME: 'linux' SETUP_CONDA: 'true'
# SETUP_CONDA: 'true' pool: lgb_agent_pool_ado
# pool: mariner-20240410-0 container: ubuntu-latest
# container: ubuntu-latest strategy:
# strategy: matrix:
# matrix: regular:
# regular: TASK: regular
# TASK: regular sdist:
# sdist: TASK: sdist
# TASK: sdist bdist:
# bdist: TASK: bdist
# TASK: bdist PYTHON_VERSION: '3.11'
# PYTHON_VERSION: '3.11' inference:
# inference: TASK: if-else
# TASK: if-else mpi_source:
# mpi_source: TASK: mpi
# TASK: mpi METHOD: source
# METHOD: source mpi_pip:
# mpi_pip: TASK: mpi
# TASK: mpi METHOD: pip
# METHOD: pip PYTHON_VERSION: '3.12'
# PYTHON_VERSION: '3.12' mpi_wheel:
# mpi_wheel: TASK: mpi
# TASK: mpi METHOD: wheel
# METHOD: wheel PYTHON_VERSION: '3.10'
# PYTHON_VERSION: '3.10' gpu_source:
# gpu_source: TASK: gpu
# TASK: gpu METHOD: source
# METHOD: source PYTHON_VERSION: '3.12'
# PYTHON_VERSION: '3.12' gpu_pip:
# gpu_pip: TASK: gpu
# TASK: gpu METHOD: pip
# METHOD: pip PYTHON_VERSION: '3.11'
# PYTHON_VERSION: '3.11' gpu_wheel:
# gpu_wheel: TASK: gpu
# TASK: gpu METHOD: wheel
# METHOD: wheel PYTHON_VERSION: '3.10'
# PYTHON_VERSION: '3.10' cpp_tests:
# cpp_tests: TASK: cpp-tests
# TASK: cpp-tests METHOD: with-sanitizers
# METHOD: with-sanitizers steps:
# steps: - script: |
# - script: | echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
# echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY" CONDA=$HOME/miniforge
# CONDA=$HOME/miniforge echo "##vso[task.setvariable variable=CONDA]$CONDA"
# echo "##vso[task.setvariable variable=CONDA]$CONDA" echo "##vso[task.prependpath]$CONDA/bin"
# echo "##vso[task.prependpath]$CONDA/bin" displayName: 'Set variables'
# displayName: 'Set variables' # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
# # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301 - script: |
# - script: | /tmp/docker exec -t -u 0 ci-container \
# /tmp/docker exec -t -u 0 ci-container \ sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
# sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo" displayName: 'Install sudo'
# displayName: 'Install sudo' - script: |
# - script: | sudo apt-get update
# sudo apt-get update sudo apt-get install -y --no-install-recommends git
# sudo apt-get install -y --no-install-recommends git git clean -d -f -x
# git clean -d -f -x displayName: 'Clean source directory'
# displayName: 'Clean source directory' - task: Bash@3
# - task: Bash@3 displayName: Setup
# displayName: Setup inputs:
# inputs: filePath: $(Build.SourcesDirectory)/.ci/setup.sh
# filePath: $(Build.SourcesDirectory)/.ci/setup.sh targetType: 'filePath'
# targetType: 'filePath' - task: Bash@3
# - task: Bash@3 displayName: Test
# displayName: Test inputs:
# inputs: filePath: $(Build.SourcesDirectory)/.ci/test.sh
# filePath: $(Build.SourcesDirectory)/.ci/test.sh targetType: 'filePath'
# targetType: 'filePath'
######### #########
# macOS # # macOS #
######### #########
...@@ -365,9 +364,8 @@ jobs: ...@@ -365,9 +364,8 @@ jobs:
########### ###########
- job: Package - job: Package
dependsOn: dependsOn:
# TODO: add jobs back when https://github.com/microsoft/LightGBM/issues/6918 is resolved - Linux
# - Linux - Linux_latest
# - Linux_latest
- macOS - macOS
- Windows - Windows
- R_artifact - R_artifact
...@@ -407,20 +405,19 @@ jobs: ...@@ -407,20 +405,19 @@ jobs:
inputs: inputs:
artifactName: R-package artifactName: R-package
downloadPath: $(Build.SourcesDirectory)/R downloadPath: $(Build.SourcesDirectory)/R
# TODO: add scripts back when https://github.com/microsoft/LightGBM/issues/6918 is resolved - script: |
# - script: | python "$(Build.SourcesDirectory)/.ci/create-nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
# python "$(Build.SourcesDirectory)/.ci/create-nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets" displayName: 'Create NuGet configuration files'
# displayName: 'Create NuGet configuration files' - task: NuGetCommand@2
# - task: NuGetCommand@2 inputs:
# inputs: command: pack
# command: pack packagesToPack: '$(Build.SourcesDirectory)/.ci/nuget/*.nuspec'
# packagesToPack: '$(Build.SourcesDirectory)/.ci/nuget/*.nuspec' packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
# packDestination: '$(Build.ArtifactStagingDirectory)/nuget' - task: PublishBuildArtifacts@1
# - task: PublishBuildArtifacts@1 inputs:
# inputs: pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
# pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget' artifactName: NuGet
# artifactName: NuGet artifactType: container
# artifactType: container
- task: GitHubRelease@0 - task: GitHubRelease@0
displayName: 'Create GitHub Release' displayName: 'Create GitHub Release'
condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v')) condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
...@@ -431,11 +428,10 @@ jobs: ...@@ -431,11 +428,10 @@ jobs:
target: '$(Build.SourceVersion)' target: '$(Build.SourceVersion)'
tagSource: 'auto' tagSource: 'auto'
title: '$(Build.SourceBranchName)' title: '$(Build.SourceBranchName)'
# TODO: add '$(Build.ArtifactStagingDirectory)/nuget/*.nupkg' artifact
# back when https://github.com/microsoft/LightGBM/issues/6918 is resolved
assets: | assets: |
$(Build.SourcesDirectory)/binaries/PackageAssets/* $(Build.SourcesDirectory)/binaries/PackageAssets/*
$(Build.SourcesDirectory)/R/R-package/* $(Build.SourcesDirectory)/R/R-package/*
$(Build.ArtifactStagingDirectory)/nuget/*.nupkg
$(Build.ArtifactStagingDirectory)/archives/* $(Build.ArtifactStagingDirectory)/archives/*
assetUploadMode: 'delete' assetUploadMode: 'delete'
isDraft: true isDraft: true
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment