.vsts-ci.yml 14.5 KB
Newer Older
1
2
3
trigger:
  branches:
    include:
4
    - master
5
6
7
  tags:
    include:
    - v*
8
9
pr:
- master
10
variables:
11
  AZURE: 'true'
12
  CMAKE_BUILD_PARALLEL_LEVEL: 4
13
  PYTHON_VERSION: '3.12'
14
15
16
17
  runCodesignValidationInjection: false
  skipComponentGovernanceDetection: true
  DOTNET_CLI_TELEMETRY_OPTOUT: true
  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
18
  SKBUILD_STRICT_CONFIG: true
19
resources:
20
21
22
23
24
25
  # The __work/ directory, where Azure DevOps writes the source files, needs to be read-write because
  # LightGBM's CI jobs write files in the source directory.
  #
  # For all the containers included here, all other directories that Azure mounts in are mounted as read-only
  # to minimize the risk of side effects from one run affecting future runs.
  # ref: https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/resources-containers-container
26
  containers:
27
  - container: linux-artifact-builder
28
    image: lightgbm/vsts-agent:manylinux_2_28_x86_64
29
30
31
32
33
    mountReadOnly:
      work: false
      externals: true
      tools: true
      tasks: true
34
  - container: ubuntu-latest
35
    image: 'ubuntu:22.04'
36
    options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
37
38
39
40
41
    mountReadOnly:
      work: false
      externals: true
      tools: true
      tasks: true
42
  - container: rbase
43
    image: wch1/r-debug
44
45
46
47
48
    mountReadOnly:
      work: false
      externals: true
      tools: true
      tasks: true
49
jobs:
Guolin Ke's avatar
Guolin Ke committed
50
###########################################
51
- job: Linux
Guolin Ke's avatar
Guolin Ke committed
52
###########################################
53
  variables:
54
    COMPILER: gcc
55
    SETUP_CONDA: 'false'
56
57
    OS_NAME: 'linux'
    PRODUCES_ARTIFACTS: 'true'
58
  pool: mariner-20240410-0
59
  container: linux-artifact-builder
60
  strategy:
Guolin Ke's avatar
Guolin Ke committed
61
62
63
    matrix:
      regular:
        TASK: regular
64
        PYTHON_VERSION: '3.10'
Guolin Ke's avatar
Guolin Ke committed
65
66
      sdist:
        TASK: sdist
67
        PYTHON_VERSION: '3.8'
Guolin Ke's avatar
Guolin Ke committed
68
69
      bdist:
        TASK: bdist
70
        PYTHON_VERSION: '3.9'
71
72
73
74
75
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
76
        PYTHON_VERSION: '3.9'
77
78
      gpu_source:
        TASK: gpu
Guolin Ke's avatar
Guolin Ke committed
79
        METHOD: source
80
81
      swig:
        TASK: swig
Guolin Ke's avatar
Guolin Ke committed
82
  steps:
83
  - script: |
84
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
85
      echo "##vso[task.prependpath]/usr/lib64/openmpi/bin"
Nikita Titov's avatar
Nikita Titov committed
86
      echo "##vso[task.prependpath]$CONDA/bin"
87
    displayName: 'Set variables'
88
89
90
  - script: |
      git clean -d -f -x
    displayName: 'Clean source directory'
91
92
93
  - script: |
      echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt'
    displayName: 'Add commit hash to artifacts archive'
94
  - task: Bash@3
95
    displayName: Setup
96
97
98
99
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/setup.sh
      targetType: filePath
  - task: Bash@3
100
    displayName: Test
101
102
103
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/test.sh
      targetType: filePath
Nikita Titov's avatar
Nikita Titov committed
104
  - task: PublishBuildArtifacts@1
105
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Nikita Titov's avatar
Nikita Titov committed
106
107
108
109
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
110
###########################################
111
112
113
- job: Linux_latest
###########################################
  variables:
114
    COMPILER: clang-17
115
    DEBIAN_FRONTEND: 'noninteractive'
116
    IN_UBUNTU_BASE_CONTAINER: 'true'
117
118
    OS_NAME: 'linux'
    SETUP_CONDA: 'true'
119
  pool: mariner-20240410-0
120
121
122
123
124
125
126
127
128
  container: ubuntu-latest
  strategy:
    matrix:
      regular:
        TASK: regular
      sdist:
        TASK: sdist
      bdist:
        TASK: bdist
129
        PYTHON_VERSION: '3.10'
130
131
132
133
134
135
136
137
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
      mpi_pip:
        TASK: mpi
        METHOD: pip
138
        PYTHON_VERSION: '3.11'
139
140
141
      mpi_wheel:
        TASK: mpi
        METHOD: wheel
142
        PYTHON_VERSION: '3.9'
143
144
145
      gpu_source:
        TASK: gpu
        METHOD: source
146
        PYTHON_VERSION: '3.11'
147
148
149
      gpu_pip:
        TASK: gpu
        METHOD: pip
150
        PYTHON_VERSION: '3.10'
151
152
153
      gpu_wheel:
        TASK: gpu
        METHOD: wheel
154
        PYTHON_VERSION: '3.9'
155
156
157
      cpp_tests:
        TASK: cpp-tests
        METHOD: with-sanitizers
158
159
160
  steps:
  - script: |
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
161
      CONDA=$HOME/miniforge
162
163
164
165
166
167
168
169
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
    displayName: 'Set variables'
  # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
  - script: |
      /tmp/docker exec -t -u 0 ci-container \
      sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
    displayName: 'Install sudo'
170
171
172
173
174
175
  - script: |
      sudo apt-get update
      sudo apt-get install -y --no-install-recommends git
      git clean -d -f -x
    displayName: 'Clean source directory'
  - task: Bash@3
176
    displayName: Setup
177
178
179
180
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/setup.sh
      targetType: 'filePath'
  - task: Bash@3
181
    displayName: Test
182
183
184
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/test.sh
      targetType: 'filePath'
185
###########################################
186
187
188
- job: QEMU_multiarch
###########################################
  variables:
189
    BUILD_DIRECTORY: /LightGBM
190
191
192
    COMPILER: gcc
    PRODUCES_ARTIFACTS: 'true'
  pool:
193
    vmImage: ubuntu-22.04
194
  timeoutInMinutes: 180
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
  strategy:
    matrix:
      bdist:
        TASK: bdist
        ARCH: aarch64
  steps:
  - script: |
      sudo apt-get update
      sudo apt-get install --no-install-recommends -y \
        binfmt-support \
        qemu \
        qemu-user \
        qemu-user-static
    displayName: 'Install QEMU'
  - script: |
      docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
    displayName: 'Enable Docker multi-architecture support'
212
213
214
  - script: |
      git clean -d -f -x
    displayName: 'Clean source directory'
215
216
  - script: |
      cat > docker-script.sh <<EOF
217
      export CONDA=\$HOME/miniforge
218
219
      export PATH=\$CONDA/bin:/opt/rh/llvm-toolset-7.0/root/usr/bin:\$PATH
      export LD_LIBRARY_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64:\$LD_LIBRARY_PATH
220
221
      \$BUILD_DIRECTORY/.ci/setup.sh || exit 1
      \$BUILD_DIRECTORY/.ci/test.sh || exit 1
222
      EOF
223
      IMAGE_URI="lightgbm/vsts-agent:manylinux2014_aarch64"
224
225
      docker pull "${IMAGE_URI}" || exit 1
      PLATFORM=$(docker inspect --format='{{.Os}}/{{.Architecture}}' "${IMAGE_URI}") || exit 1
226
      echo "detected image platform: ${PLATFORM}"
227
      docker run \
228
        --platform "${PLATFORM}" \
229
        --rm \
230
231
232
233
234
235
236
237
238
239
        --env AZURE=true \
        --env BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY \
        --env BUILD_DIRECTORY=$BUILD_DIRECTORY \
        --env COMPILER=$COMPILER \
        --env METHOD=$METHOD \
        --env OS_NAME=linux \
        --env PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS \
        --env PYTHON_VERSION=$PYTHON_VERSION \
        --env TASK=$TASK \
        -v "$(Build.SourcesDirectory)":"$BUILD_DIRECTORY" \
240
        -v "$(Build.ArtifactStagingDirectory)":"$(Build.ArtifactStagingDirectory)" \
241
        "${IMAGE_URI}" \
242
        /bin/bash $BUILD_DIRECTORY/docker-script.sh
243
244
245
246
247
248
249
250
    displayName: 'Setup and run tests'
  - task: PublishBuildArtifacts@1
    condition: and(succeeded(), in(variables['TASK'], 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
###########################################
251
- job: macOS
Guolin Ke's avatar
Guolin Ke committed
252
###########################################
253
  variables:
254
    COMPILER: clang
255
256
    OS_NAME: 'macos'
    PRODUCES_ARTIFACTS: 'true'
257
  pool:
258
    vmImage: 'macOS-12'
259
  strategy:
Guolin Ke's avatar
Guolin Ke committed
260
261
262
    matrix:
      regular:
        TASK: regular
263
        PYTHON_VERSION: '3.10'
Guolin Ke's avatar
Guolin Ke committed
264
265
      sdist:
        TASK: sdist
266
        PYTHON_VERSION: '3.9'
Guolin Ke's avatar
Guolin Ke committed
267
268
      bdist:
        TASK: bdist
269
270
      swig:
        TASK: swig
271
272
      cpp_tests:
        TASK: cpp-tests
273
274
        METHOD: with-sanitizers
        SANITIZERS: "address;undefined"
Guolin Ke's avatar
Guolin Ke committed
275
  steps:
276
  - script: |
277
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
278
      CONDA=$AGENT_HOMEDIRECTORY/miniforge
Nikita Titov's avatar
Nikita Titov committed
279
280
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
281
      echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64"
282
    displayName: 'Set variables'
283
284
285
286
  - script: |
      git clean -d -f -x
    displayName: 'Clean source directory'
  - task: Bash@3
287
    displayName: Setup
288
289
290
291
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/setup.sh
      targetType: filePath
  - task: Bash@3
292
    displayName: Test
293
294
295
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/test.sh
      targetType: filePath
Guolin Ke's avatar
Guolin Ke committed
296
  - task: PublishBuildArtifacts@1
297
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
298
    inputs:
299
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
300
301
302
      artifactName: PackageAssets
      artifactType: container
###########################################
303
- job: Windows
Guolin Ke's avatar
Guolin Ke committed
304
###########################################
305
  pool:
306
    vmImage: 'windows-2019'
307
  strategy:
Guolin Ke's avatar
Guolin Ke committed
308
309
310
    matrix:
      regular:
        TASK: regular
311
        PYTHON_VERSION: '3.10'
Guolin Ke's avatar
Guolin Ke committed
312
313
      sdist:
        TASK: sdist
314
        PYTHON_VERSION: '3.9'
Guolin Ke's avatar
Guolin Ke committed
315
316
      bdist:
        TASK: bdist
317
318
      swig:
        TASK: swig
319
320
      cpp_tests:
        TASK: cpp-tests
Guolin Ke's avatar
Guolin Ke committed
321
  steps:
322
323
324
  - powershell: |
      Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
    displayName: 'Set Variables'
325
326
327
  - script: |
      git clean -d -f -x
    displayName: 'Clean source directory'
328
329
  - script: |
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install_opencl.ps1"
330
    condition: eq(variables['TASK'], 'bdist')
331
    displayName: 'Install OpenCL'
Nikita Titov's avatar
Nikita Titov committed
332
  - script: |
333
334
      cmd /c "conda config --remove channels defaults"
      cmd /c "conda config --add channels nodefaults"
335
336
      cmd /c "conda config --add channels conda-forge"
      cmd /c "conda config --set channel_priority strict"
337
338
      cmd /c "conda init powershell"
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1"
339
    displayName: Test
Guolin Ke's avatar
Guolin Ke committed
340
  - task: PublishBuildArtifacts@1
341
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
342
    inputs:
343
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
344
345
      artifactName: PackageAssets
      artifactType: container
346
347
348
349
350
###########################################
- job: R_artifact
###########################################
  condition: not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
  pool:
351
    vmImage: 'ubuntu-22.04'
352
353
  container: rbase
  steps:
354
355
356
  - script: |
      git clean -d -f -x
    displayName: 'Clean source directory'
357
358
  - script: |
      LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g")
359
360
361
      R_LIB_PATH=~/Rlib
      export R_LIBS=${R_LIB_PATH}
      mkdir -p ${R_LIB_PATH}
362
363
      RDscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl'),  lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" || exit 1
      sh build-cran-package.sh --r-executable=RD || exit 1
364
365
366
367
368
369
370
371
      mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz
    displayName: 'Build CRAN R-package'
  - task: PublishBuildArtifacts@1
    condition: succeeded()
    inputs:
      pathtoPublish: $(Build.ArtifactStagingDirectory)
      artifactName: R-package
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
372

373
###########################################
374
- job: Package
375
###########################################
Guolin Ke's avatar
Guolin Ke committed
376
377
  dependsOn:
  - Linux
378
  - Linux_latest
379
  - QEMU_multiarch
380
  - macOS
Guolin Ke's avatar
Guolin Ke committed
381
  - Windows
382
  - R_artifact
383
  condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
384
  pool:
385
    vmImage: 'ubuntu-22.04'
Guolin Ke's avatar
Guolin Ke committed
386
  steps:
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
  # Create archives with complete source code included (with git submodules)
  - task: ArchiveFiles@2
    displayName: Create zip archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: zip
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_zip.zip'
      replaceExistingArchive: true
  - task: ArchiveFiles@2
    displayName: Create tar.gz archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: tar
      tarCompression: gz
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_tar_gz.tar.gz'
      replaceExistingArchive: true
Guolin Ke's avatar
Guolin Ke committed
407
408
409
410
411
412
  # Download all agent packages from all previous phases
  - task: DownloadBuildArtifacts@0
    displayName: Download package assets
    inputs:
      artifactName: PackageAssets
      downloadPath: $(Build.SourcesDirectory)/binaries
413
414
415
416
417
418
  - task: DownloadBuildArtifacts@0
    displayName: Download R-package
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      artifactName: R-package
      downloadPath: $(Build.SourcesDirectory)/R
Guolin Ke's avatar
Guolin Ke committed
419
  - script: |
420
      python "$(Build.SourcesDirectory)/.nuget/create_nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
421
422
423
424
425
    displayName: 'Create NuGet configuration files'
  - task: NuGetCommand@2
    inputs:
      command: pack
      packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec'
426
      packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
Guolin Ke's avatar
Guolin Ke committed
427
428
  - task: PublishBuildArtifacts@1
    inputs:
429
      pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
430
      artifactName: NuGet
Guolin Ke's avatar
Guolin Ke committed
431
      artifactType: container
432
433
434
435
436
437
438
439
440
441
442
443
  - task: GitHubRelease@0
    displayName: 'Create GitHub Release'
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      gitHubConnection: guolinke
      repositoryName: '$(Build.Repository.Name)'
      action: 'create'
      target: '$(Build.SourceVersion)'
      tagSource: 'auto'
      title: '$(Build.SourceBranchName)'
      assets: |
        $(Build.SourcesDirectory)/binaries/PackageAssets/*
444
        $(Build.SourcesDirectory)/R/R-package/*
445
446
        $(Build.ArtifactStagingDirectory)/nuget/*.nupkg
        $(Build.ArtifactStagingDirectory)/archives/*
447
448
449
      assetUploadMode: 'delete'
      isDraft: true
      isPreRelease: false
Guolin Ke's avatar
Guolin Ke committed
450
      addChangeLog: false