.vsts-ci.yml 14.5 KB
Newer Older
1
2
3
trigger:
  branches:
    include:
4
    - master
5
6
7
  tags:
    include:
    - v*
8
9
pr:
- master
10
variables:
11
  AZURE: 'true'
12
  CMAKE_BUILD_PARALLEL_LEVEL: 4
13
  PYTHON_VERSION: '3.12'
14
15
16
17
  runCodesignValidationInjection: false
  skipComponentGovernanceDetection: true
  DOTNET_CLI_TELEMETRY_OPTOUT: true
  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
18
resources:
19
20
21
22
23
24
  # The __work/ directory, where Azure DevOps writes the source files, needs to be read-write because
  # LightGBM's CI jobs write files in the source directory.
  #
  # For all the containers included here, all other directories that Azure mounts in are mounted as read-only
  # to minimize the risk of side effects from one run affecting future runs.
  # ref: https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/resources-containers-container
25
  containers:
26
  - container: linux-artifact-builder
27
    image: lightgbm/vsts-agent:manylinux_2_28_x86_64
28
29
30
31
32
    mountReadOnly:
      work: false
      externals: true
      tools: true
      tasks: true
33
  - container: ubuntu-latest
34
    image: 'ubuntu:22.04'
35
    options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
36
37
38
39
40
    mountReadOnly:
      work: false
      externals: true
      tools: true
      tasks: true
41
  - container: rbase
42
    image: wch1/r-debug
43
44
45
46
47
    mountReadOnly:
      work: false
      externals: true
      tools: true
      tasks: true
48
jobs:
Guolin Ke's avatar
Guolin Ke committed
49
###########################################
50
- job: Linux
Guolin Ke's avatar
Guolin Ke committed
51
###########################################
52
  variables:
53
    COMPILER: gcc
54
    SETUP_CONDA: 'false'
55
56
    OS_NAME: 'linux'
    PRODUCES_ARTIFACTS: 'true'
57
  pool: mariner-20240410-0
58
  container: linux-artifact-builder
59
  strategy:
Guolin Ke's avatar
Guolin Ke committed
60
61
62
    matrix:
      regular:
        TASK: regular
63
        PYTHON_VERSION: '3.10'
Guolin Ke's avatar
Guolin Ke committed
64
65
      sdist:
        TASK: sdist
66
        PYTHON_VERSION: '3.8'
Guolin Ke's avatar
Guolin Ke committed
67
68
      bdist:
        TASK: bdist
69
        PYTHON_VERSION: '3.9'
70
71
72
73
74
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
75
        PYTHON_VERSION: '3.9'
76
77
      gpu_source:
        TASK: gpu
Guolin Ke's avatar
Guolin Ke committed
78
        METHOD: source
79
80
      swig:
        TASK: swig
Guolin Ke's avatar
Guolin Ke committed
81
  steps:
82
  - script: |
83
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
84
      echo "##vso[task.prependpath]/usr/lib64/openmpi/bin"
Nikita Titov's avatar
Nikita Titov committed
85
      echo "##vso[task.prependpath]$CONDA/bin"
86
    displayName: 'Set variables'
87
88
89
  - script: |
      git clean -d -f -x
    displayName: 'Clean source directory'
90
91
92
  - script: |
      echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt'
    displayName: 'Add commit hash to artifacts archive'
93
  - task: Bash@3
94
    displayName: Setup
95
96
97
98
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/setup.sh
      targetType: filePath
  - task: Bash@3
99
    displayName: Test
100
101
102
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/test.sh
      targetType: filePath
Nikita Titov's avatar
Nikita Titov committed
103
  - task: PublishBuildArtifacts@1
104
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Nikita Titov's avatar
Nikita Titov committed
105
106
107
108
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
109
###########################################
110
111
112
- job: Linux_latest
###########################################
  variables:
113
    COMPILER: clang-17
114
    DEBIAN_FRONTEND: 'noninteractive'
115
    IN_UBUNTU_BASE_CONTAINER: 'true'
116
117
    OS_NAME: 'linux'
    SETUP_CONDA: 'true'
118
  pool: mariner-20240410-0
119
120
121
122
123
124
125
126
127
  container: ubuntu-latest
  strategy:
    matrix:
      regular:
        TASK: regular
      sdist:
        TASK: sdist
      bdist:
        TASK: bdist
128
        PYTHON_VERSION: '3.10'
129
130
131
132
133
134
135
136
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
      mpi_pip:
        TASK: mpi
        METHOD: pip
137
        PYTHON_VERSION: '3.11'
138
139
140
      mpi_wheel:
        TASK: mpi
        METHOD: wheel
141
        PYTHON_VERSION: '3.9'
142
143
144
      gpu_source:
        TASK: gpu
        METHOD: source
145
        PYTHON_VERSION: '3.11'
146
147
148
      gpu_pip:
        TASK: gpu
        METHOD: pip
149
        PYTHON_VERSION: '3.10'
150
151
152
      gpu_wheel:
        TASK: gpu
        METHOD: wheel
153
        PYTHON_VERSION: '3.9'
154
155
156
      cpp_tests:
        TASK: cpp-tests
        METHOD: with-sanitizers
157
158
159
  steps:
  - script: |
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
160
      CONDA=$HOME/miniforge
161
162
163
164
165
166
167
168
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
    displayName: 'Set variables'
  # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
  - script: |
      /tmp/docker exec -t -u 0 ci-container \
      sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
    displayName: 'Install sudo'
169
170
171
172
173
174
  - script: |
      sudo apt-get update
      sudo apt-get install -y --no-install-recommends git
      git clean -d -f -x
    displayName: 'Clean source directory'
  - task: Bash@3
175
    displayName: Setup
176
177
178
179
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/setup.sh
      targetType: 'filePath'
  - task: Bash@3
180
    displayName: Test
181
182
183
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/test.sh
      targetType: 'filePath'
184
###########################################
185
186
187
- job: QEMU_multiarch
###########################################
  variables:
188
    BUILD_DIRECTORY: /LightGBM
189
190
191
    COMPILER: gcc
    PRODUCES_ARTIFACTS: 'true'
  pool:
192
    vmImage: ubuntu-22.04
193
  timeoutInMinutes: 180
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
  strategy:
    matrix:
      bdist:
        TASK: bdist
        ARCH: aarch64
  steps:
  - script: |
      sudo apt-get update
      sudo apt-get install --no-install-recommends -y \
        binfmt-support \
        qemu \
        qemu-user \
        qemu-user-static
    displayName: 'Install QEMU'
  - script: |
      docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
    displayName: 'Enable Docker multi-architecture support'
211
212
213
  - script: |
      git clean -d -f -x
    displayName: 'Clean source directory'
214
215
  - script: |
      cat > docker-script.sh <<EOF
216
      export CONDA=\$HOME/miniforge
217
218
      export PATH=\$CONDA/bin:/opt/rh/llvm-toolset-7.0/root/usr/bin:\$PATH
      export LD_LIBRARY_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64:\$LD_LIBRARY_PATH
219
220
      \$BUILD_DIRECTORY/.ci/setup.sh || exit 1
      \$BUILD_DIRECTORY/.ci/test.sh || exit 1
221
      EOF
222
      IMAGE_URI="lightgbm/vsts-agent:manylinux2014_aarch64"
223
224
      docker pull "${IMAGE_URI}" || exit 1
      PLATFORM=$(docker inspect --format='{{.Os}}/{{.Architecture}}' "${IMAGE_URI}") || exit 1
225
      echo "detected image platform: ${PLATFORM}"
226
      docker run \
227
        --platform "${PLATFORM}" \
228
        --rm \
229
230
231
232
233
234
235
236
237
238
        --env AZURE=true \
        --env BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY \
        --env BUILD_DIRECTORY=$BUILD_DIRECTORY \
        --env COMPILER=$COMPILER \
        --env METHOD=$METHOD \
        --env OS_NAME=linux \
        --env PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS \
        --env PYTHON_VERSION=$PYTHON_VERSION \
        --env TASK=$TASK \
        -v "$(Build.SourcesDirectory)":"$BUILD_DIRECTORY" \
239
        -v "$(Build.ArtifactStagingDirectory)":"$(Build.ArtifactStagingDirectory)" \
240
        "${IMAGE_URI}" \
241
        /bin/bash $BUILD_DIRECTORY/docker-script.sh
242
243
244
245
246
247
248
249
    displayName: 'Setup and run tests'
  - task: PublishBuildArtifacts@1
    condition: and(succeeded(), in(variables['TASK'], 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
###########################################
250
- job: macOS
Guolin Ke's avatar
Guolin Ke committed
251
###########################################
252
  variables:
253
    COMPILER: clang
254
255
    OS_NAME: 'macos'
    PRODUCES_ARTIFACTS: 'true'
256
  pool:
257
    vmImage: 'macOS-11'
258
  strategy:
Guolin Ke's avatar
Guolin Ke committed
259
260
261
    matrix:
      regular:
        TASK: regular
262
        PYTHON_VERSION: '3.10'
Guolin Ke's avatar
Guolin Ke committed
263
264
      sdist:
        TASK: sdist
265
        PYTHON_VERSION: '3.9'
Guolin Ke's avatar
Guolin Ke committed
266
267
      bdist:
        TASK: bdist
268
269
      swig:
        TASK: swig
270
271
      cpp_tests:
        TASK: cpp-tests
272
273
        METHOD: with-sanitizers
        SANITIZERS: "address;undefined"
Guolin Ke's avatar
Guolin Ke committed
274
  steps:
275
  - script: |
276
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
277
      CONDA=$AGENT_HOMEDIRECTORY/miniforge
Nikita Titov's avatar
Nikita Titov committed
278
279
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
280
      echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64"
281
    displayName: 'Set variables'
282
283
284
285
  - script: |
      git clean -d -f -x
    displayName: 'Clean source directory'
  - task: Bash@3
286
    displayName: Setup
287
288
289
290
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/setup.sh
      targetType: filePath
  - task: Bash@3
291
    displayName: Test
292
293
294
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/test.sh
      targetType: filePath
Guolin Ke's avatar
Guolin Ke committed
295
  - task: PublishBuildArtifacts@1
296
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
297
    inputs:
298
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
299
300
301
      artifactName: PackageAssets
      artifactType: container
###########################################
302
- job: Windows
Guolin Ke's avatar
Guolin Ke committed
303
###########################################
304
  pool:
305
    vmImage: 'windows-2019'
306
  strategy:
Guolin Ke's avatar
Guolin Ke committed
307
308
309
    matrix:
      regular:
        TASK: regular
310
        PYTHON_VERSION: '3.10'
Guolin Ke's avatar
Guolin Ke committed
311
312
      sdist:
        TASK: sdist
313
        PYTHON_VERSION: '3.9'
Guolin Ke's avatar
Guolin Ke committed
314
315
      bdist:
        TASK: bdist
316
317
      swig:
        TASK: swig
318
319
      cpp_tests:
        TASK: cpp-tests
Guolin Ke's avatar
Guolin Ke committed
320
  steps:
321
322
323
  - powershell: |
      Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
    displayName: 'Set Variables'
324
325
326
  - script: |
      git clean -d -f -x
    displayName: 'Clean source directory'
327
328
  - script: |
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install_opencl.ps1"
329
    condition: eq(variables['TASK'], 'bdist')
330
    displayName: 'Install OpenCL'
Nikita Titov's avatar
Nikita Titov committed
331
  - script: |
332
333
      cmd /c "conda config --remove channels defaults"
      cmd /c "conda config --add channels nodefaults"
334
335
      cmd /c "conda config --add channels conda-forge"
      cmd /c "conda config --set channel_priority strict"
336
337
      cmd /c "conda init powershell"
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1"
338
    displayName: Test
Guolin Ke's avatar
Guolin Ke committed
339
  - task: PublishBuildArtifacts@1
340
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
341
    inputs:
342
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
343
344
      artifactName: PackageAssets
      artifactType: container
345
346
347
348
349
###########################################
- job: R_artifact
###########################################
  condition: not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
  pool:
350
    vmImage: 'ubuntu-22.04'
351
352
  container: rbase
  steps:
353
354
355
  - script: |
      git clean -d -f -x
    displayName: 'Clean source directory'
356
357
  - script: |
      LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g")
358
359
360
      R_LIB_PATH=~/Rlib
      export R_LIBS=${R_LIB_PATH}
      mkdir -p ${R_LIB_PATH}
361
362
      RDscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl'),  lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" || exit 1
      sh build-cran-package.sh --r-executable=RD || exit 1
363
364
365
366
367
368
369
370
      mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz
    displayName: 'Build CRAN R-package'
  - task: PublishBuildArtifacts@1
    condition: succeeded()
    inputs:
      pathtoPublish: $(Build.ArtifactStagingDirectory)
      artifactName: R-package
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
371

372
###########################################
373
- job: Package
374
###########################################
Guolin Ke's avatar
Guolin Ke committed
375
376
  dependsOn:
  - Linux
377
  - Linux_latest
378
  - QEMU_multiarch
379
  - macOS
Guolin Ke's avatar
Guolin Ke committed
380
  - Windows
381
  - R_artifact
382
  condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
383
  pool:
384
    vmImage: 'ubuntu-22.04'
Guolin Ke's avatar
Guolin Ke committed
385
  steps:
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
  # Create archives with complete source code included (with git submodules)
  - task: ArchiveFiles@2
    displayName: Create zip archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: zip
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_zip.zip'
      replaceExistingArchive: true
  - task: ArchiveFiles@2
    displayName: Create tar.gz archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: tar
      tarCompression: gz
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_tar_gz.tar.gz'
      replaceExistingArchive: true
Guolin Ke's avatar
Guolin Ke committed
406
407
408
409
410
411
  # Download all agent packages from all previous phases
  - task: DownloadBuildArtifacts@0
    displayName: Download package assets
    inputs:
      artifactName: PackageAssets
      downloadPath: $(Build.SourcesDirectory)/binaries
412
413
414
415
416
417
  - task: DownloadBuildArtifacts@0
    displayName: Download R-package
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      artifactName: R-package
      downloadPath: $(Build.SourcesDirectory)/R
Guolin Ke's avatar
Guolin Ke committed
418
  - script: |
419
      python "$(Build.SourcesDirectory)/.nuget/create_nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
420
421
422
423
424
    displayName: 'Create NuGet configuration files'
  - task: NuGetCommand@2
    inputs:
      command: pack
      packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec'
425
      packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
Guolin Ke's avatar
Guolin Ke committed
426
427
  - task: PublishBuildArtifacts@1
    inputs:
428
      pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
429
      artifactName: NuGet
Guolin Ke's avatar
Guolin Ke committed
430
      artifactType: container
431
432
433
434
435
436
437
438
439
440
441
442
  - task: GitHubRelease@0
    displayName: 'Create GitHub Release'
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      gitHubConnection: guolinke
      repositoryName: '$(Build.Repository.Name)'
      action: 'create'
      target: '$(Build.SourceVersion)'
      tagSource: 'auto'
      title: '$(Build.SourceBranchName)'
      assets: |
        $(Build.SourcesDirectory)/binaries/PackageAssets/*
443
        $(Build.SourcesDirectory)/R/R-package/*
444
445
        $(Build.ArtifactStagingDirectory)/nuget/*.nupkg
        $(Build.ArtifactStagingDirectory)/archives/*
446
447
448
      assetUploadMode: 'delete'
      isDraft: true
      isPreRelease: false
Guolin Ke's avatar
Guolin Ke committed
449
      addChangeLog: false