.vsts-ci.yml 14.6 KB
Newer Older
1
2
3
trigger:
  branches:
    include:
4
    - master
5
6
7
  tags:
    include:
    - v*
8
9
pr:
- master
10
variables:
11
  AZURE: 'true'
12
  CMAKE_BUILD_PARALLEL_LEVEL: 4
13
  PYTHON_VERSION: '3.12'
14
15
  runCodesignValidationInjection: false
  skipComponentGovernanceDetection: true
16
17
  Codeql.Enabled: false
  Codeql.SkipTaskAutoInjection: true
18
19
  DOTNET_CLI_TELEMETRY_OPTOUT: true
  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
20
  SKBUILD_STRICT_CONFIG: true
21
resources:
22
23
24
25
26
27
  # The __work/ directory, where Azure DevOps writes the source files, needs to be read-write because
  # LightGBM's CI jobs write files in the source directory.
  #
  # For all the containers included here, all other directories that Azure mounts in are mounted as read-only
  # to minimize the risk of side effects from one run affecting future runs.
  # ref: https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/resources-containers-container
28
  containers:
29
  - container: linux-artifact-builder
30
    image: lightgbm/vsts-agent:manylinux_2_28_x86_64
31
32
33
34
35
    mountReadOnly:
      work: false
      externals: true
      tools: true
      tasks: true
36
  - container: ubuntu-latest
37
    image: 'ubuntu:22.04'
38
    options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
39
40
41
42
43
    mountReadOnly:
      work: false
      externals: true
      tools: true
      tasks: true
44
  - container: rbase
45
    image: wch1/r-debug
46
47
48
49
50
    mountReadOnly:
      work: false
      externals: true
      tools: true
      tasks: true
51
jobs:
Guolin Ke's avatar
Guolin Ke committed
52
###########################################
53
- job: Linux
Guolin Ke's avatar
Guolin Ke committed
54
###########################################
55
  variables:
56
    COMPILER: gcc
57
    SETUP_CONDA: 'false'
58
59
    OS_NAME: 'linux'
    PRODUCES_ARTIFACTS: 'true'
60
  pool: mariner-20240410-0
61
  container: linux-artifact-builder
62
  strategy:
Guolin Ke's avatar
Guolin Ke committed
63
64
65
    matrix:
      regular:
        TASK: regular
66
        PYTHON_VERSION: '3.10'
Guolin Ke's avatar
Guolin Ke committed
67
68
      sdist:
        TASK: sdist
69
        PYTHON_VERSION: '3.8'
Guolin Ke's avatar
Guolin Ke committed
70
71
      bdist:
        TASK: bdist
72
        PYTHON_VERSION: '3.9'
73
74
75
76
77
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
78
        PYTHON_VERSION: '3.9'
79
80
      gpu_source:
        TASK: gpu
Guolin Ke's avatar
Guolin Ke committed
81
        METHOD: source
82
83
      swig:
        TASK: swig
Guolin Ke's avatar
Guolin Ke committed
84
  steps:
85
  - script: |
86
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
87
      echo "##vso[task.prependpath]/usr/lib64/openmpi/bin"
Nikita Titov's avatar
Nikita Titov committed
88
      echo "##vso[task.prependpath]$CONDA/bin"
89
    displayName: 'Set variables'
90
91
92
  - script: |
      git clean -d -f -x
    displayName: 'Clean source directory'
93
94
95
  - script: |
      echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt'
    displayName: 'Add commit hash to artifacts archive'
96
  - task: Bash@3
97
    displayName: Setup
98
99
100
101
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/setup.sh
      targetType: filePath
  - task: Bash@3
102
    displayName: Test
103
104
105
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/test.sh
      targetType: filePath
Nikita Titov's avatar
Nikita Titov committed
106
  - task: PublishBuildArtifacts@1
107
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Nikita Titov's avatar
Nikita Titov committed
108
109
110
111
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
112
###########################################
113
114
115
- job: Linux_latest
###########################################
  variables:
116
    COMPILER: clang-17
117
    DEBIAN_FRONTEND: 'noninteractive'
118
    IN_UBUNTU_BASE_CONTAINER: 'true'
119
120
    OS_NAME: 'linux'
    SETUP_CONDA: 'true'
121
  pool: mariner-20240410-0
122
123
124
125
126
127
128
129
130
  container: ubuntu-latest
  strategy:
    matrix:
      regular:
        TASK: regular
      sdist:
        TASK: sdist
      bdist:
        TASK: bdist
131
        PYTHON_VERSION: '3.10'
132
133
134
135
136
137
138
139
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
      mpi_pip:
        TASK: mpi
        METHOD: pip
140
        PYTHON_VERSION: '3.11'
141
142
143
      mpi_wheel:
        TASK: mpi
        METHOD: wheel
144
        PYTHON_VERSION: '3.9'
145
146
147
      gpu_source:
        TASK: gpu
        METHOD: source
148
        PYTHON_VERSION: '3.11'
149
150
151
      gpu_pip:
        TASK: gpu
        METHOD: pip
152
        PYTHON_VERSION: '3.10'
153
154
155
      gpu_wheel:
        TASK: gpu
        METHOD: wheel
156
        PYTHON_VERSION: '3.9'
157
158
159
      cpp_tests:
        TASK: cpp-tests
        METHOD: with-sanitizers
160
161
162
  steps:
  - script: |
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
163
      CONDA=$HOME/miniforge
164
165
166
167
168
169
170
171
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
    displayName: 'Set variables'
  # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
  - script: |
      /tmp/docker exec -t -u 0 ci-container \
      sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
    displayName: 'Install sudo'
172
173
174
175
176
177
  - script: |
      sudo apt-get update
      sudo apt-get install -y --no-install-recommends git
      git clean -d -f -x
    displayName: 'Clean source directory'
  - task: Bash@3
178
    displayName: Setup
179
180
181
182
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/setup.sh
      targetType: 'filePath'
  - task: Bash@3
183
    displayName: Test
184
185
186
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/test.sh
      targetType: 'filePath'
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
###########################################
- job: QEMU_multiarch
###########################################
  variables:
    BUILD_DIRECTORY: /LightGBM
    COMPILER: gcc
    PRODUCES_ARTIFACTS: 'true'
  pool:
    vmImage: ubuntu-22.04
  timeoutInMinutes: 180
  strategy:
    matrix:
      bdist:
        TASK: bdist
        ARCH: aarch64
  steps:
  - script: |
      sudo apt-get update
      sudo apt-get install --no-install-recommends -y \
        binfmt-support \
        qemu \
        qemu-user \
        qemu-user-static
    displayName: 'Install QEMU'
  - script: |
      docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
    displayName: 'Enable Docker multi-architecture support'
  - script: |
      git clean -d -f -x
    displayName: 'Clean source directory'
  - script: |
      cat > docker-script.sh <<EOF
      export CONDA=\$HOME/miniforge
      export PATH=\$CONDA/bin:/opt/rh/llvm-toolset-7.0/root/usr/bin:\$PATH
      export LD_LIBRARY_PATH=/opt/rh/llvm-toolset-7.0/root/usr/lib64:\$LD_LIBRARY_PATH
      \$BUILD_DIRECTORY/.ci/setup.sh || exit 1
      \$BUILD_DIRECTORY/.ci/test.sh || exit 1
      EOF
      IMAGE_URI="lightgbm/vsts-agent:manylinux2014_aarch64"
      docker pull "${IMAGE_URI}" || exit 1
      PLATFORM=$(docker inspect --format='{{.Os}}/{{.Architecture}}' "${IMAGE_URI}") || exit 1
      echo "detected image platform: ${PLATFORM}"
      docker run \
        --platform "${PLATFORM}" \
        --rm \
        --env AZURE=true \
        --env BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY \
        --env BUILD_DIRECTORY=$BUILD_DIRECTORY \
        --env COMPILER=$COMPILER \
        --env METHOD=$METHOD \
        --env OS_NAME=linux \
        --env PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS \
        --env PYTHON_VERSION=$PYTHON_VERSION \
        --env TASK=$TASK \
        -v "$(Build.SourcesDirectory)":"$BUILD_DIRECTORY" \
        -v "$(Build.ArtifactStagingDirectory)":"$(Build.ArtifactStagingDirectory)" \
        "${IMAGE_URI}" \
        /bin/bash $BUILD_DIRECTORY/docker-script.sh
    displayName: 'Setup and run tests'
  - task: PublishBuildArtifacts@1
    condition: and(succeeded(), in(variables['TASK'], 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
252
###########################################
253
- job: macOS
Guolin Ke's avatar
Guolin Ke committed
254
###########################################
255
  variables:
256
    COMPILER: clang
257
258
    OS_NAME: 'macos'
    PRODUCES_ARTIFACTS: 'true'
259
  pool:
260
    vmImage: 'macOS-12'
261
  strategy:
Guolin Ke's avatar
Guolin Ke committed
262
263
264
    matrix:
      regular:
        TASK: regular
265
        PYTHON_VERSION: '3.10'
Guolin Ke's avatar
Guolin Ke committed
266
267
      sdist:
        TASK: sdist
268
        PYTHON_VERSION: '3.9'
Guolin Ke's avatar
Guolin Ke committed
269
270
      bdist:
        TASK: bdist
271
272
      swig:
        TASK: swig
273
274
      cpp_tests:
        TASK: cpp-tests
275
276
        METHOD: with-sanitizers
        SANITIZERS: "address;undefined"
Guolin Ke's avatar
Guolin Ke committed
277
  steps:
278
  - script: |
279
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
280
      CONDA=$AGENT_HOMEDIRECTORY/miniforge
Nikita Titov's avatar
Nikita Titov committed
281
282
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
283
      echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64"
284
    displayName: 'Set variables'
285
286
287
288
  - script: |
      git clean -d -f -x
    displayName: 'Clean source directory'
  - task: Bash@3
289
    displayName: Setup
290
291
292
293
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/setup.sh
      targetType: filePath
  - task: Bash@3
294
    displayName: Test
295
296
297
    inputs:
      filePath: $(Build.SourcesDirectory)/.ci/test.sh
      targetType: filePath
Guolin Ke's avatar
Guolin Ke committed
298
  - task: PublishBuildArtifacts@1
299
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
300
    inputs:
301
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
302
303
304
      artifactName: PackageAssets
      artifactType: container
###########################################
305
- job: Windows
Guolin Ke's avatar
Guolin Ke committed
306
###########################################
307
  pool:
308
    vmImage: 'windows-2019'
309
  strategy:
Guolin Ke's avatar
Guolin Ke committed
310
311
312
    matrix:
      regular:
        TASK: regular
313
        PYTHON_VERSION: '3.10'
Guolin Ke's avatar
Guolin Ke committed
314
315
      sdist:
        TASK: sdist
316
        PYTHON_VERSION: '3.9'
Guolin Ke's avatar
Guolin Ke committed
317
318
      bdist:
        TASK: bdist
319
320
      swig:
        TASK: swig
321
322
      cpp_tests:
        TASK: cpp-tests
Guolin Ke's avatar
Guolin Ke committed
323
  steps:
324
325
326
  - powershell: |
      Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
    displayName: 'Set Variables'
327
328
329
  - script: |
      git clean -d -f -x
    displayName: 'Clean source directory'
330
331
  - script: |
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install_opencl.ps1"
332
    condition: eq(variables['TASK'], 'bdist')
333
    displayName: 'Install OpenCL'
Nikita Titov's avatar
Nikita Titov committed
334
  - script: |
335
336
      cmd /c "conda config --remove channels defaults"
      cmd /c "conda config --add channels nodefaults"
337
338
      cmd /c "conda config --add channels conda-forge"
      cmd /c "conda config --set channel_priority strict"
339
340
      cmd /c "conda init powershell"
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1"
341
    displayName: Test
Guolin Ke's avatar
Guolin Ke committed
342
  - task: PublishBuildArtifacts@1
343
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
344
    inputs:
345
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
346
347
      artifactName: PackageAssets
      artifactType: container
348
349
350
351
352
###########################################
- job: R_artifact
###########################################
  condition: not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
  pool:
353
    vmImage: 'ubuntu-22.04'
354
355
  container: rbase
  steps:
356
357
358
  - script: |
      git clean -d -f -x
    displayName: 'Clean source directory'
359
360
  - script: |
      LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g")
361
362
363
      R_LIB_PATH=~/Rlib
      export R_LIBS=${R_LIB_PATH}
      mkdir -p ${R_LIB_PATH}
364
365
      RDscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'RhpcBLASctl'),  lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" || exit 1
      sh build-cran-package.sh --r-executable=RD || exit 1
366
367
368
369
370
371
372
373
      mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz
    displayName: 'Build CRAN R-package'
  - task: PublishBuildArtifacts@1
    condition: succeeded()
    inputs:
      pathtoPublish: $(Build.ArtifactStagingDirectory)
      artifactName: R-package
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
374

375
###########################################
376
- job: Package
377
###########################################
Guolin Ke's avatar
Guolin Ke committed
378
379
  dependsOn:
  - Linux
380
  - Linux_latest
381
  - QEMU_multiarch
382
  - macOS
Guolin Ke's avatar
Guolin Ke committed
383
  - Windows
384
  - R_artifact
385
  condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
386
  pool:
387
    vmImage: 'ubuntu-22.04'
Guolin Ke's avatar
Guolin Ke committed
388
  steps:
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
  # Create archives with complete source code included (with git submodules)
  - task: ArchiveFiles@2
    displayName: Create zip archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: zip
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_zip.zip'
      replaceExistingArchive: true
  - task: ArchiveFiles@2
    displayName: Create tar.gz archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: tar
      tarCompression: gz
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_tar_gz.tar.gz'
      replaceExistingArchive: true
Guolin Ke's avatar
Guolin Ke committed
409
410
411
412
413
414
  # Download all agent packages from all previous phases
  - task: DownloadBuildArtifacts@0
    displayName: Download package assets
    inputs:
      artifactName: PackageAssets
      downloadPath: $(Build.SourcesDirectory)/binaries
415
416
417
418
419
420
  - task: DownloadBuildArtifacts@0
    displayName: Download R-package
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      artifactName: R-package
      downloadPath: $(Build.SourcesDirectory)/R
Guolin Ke's avatar
Guolin Ke committed
421
  - script: |
422
      python "$(Build.SourcesDirectory)/.nuget/create_nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
423
424
425
426
427
    displayName: 'Create NuGet configuration files'
  - task: NuGetCommand@2
    inputs:
      command: pack
      packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec'
428
      packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
Guolin Ke's avatar
Guolin Ke committed
429
430
  - task: PublishBuildArtifacts@1
    inputs:
431
      pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
432
      artifactName: NuGet
Guolin Ke's avatar
Guolin Ke committed
433
      artifactType: container
434
435
436
437
438
439
440
441
442
443
444
445
  - task: GitHubRelease@0
    displayName: 'Create GitHub Release'
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      gitHubConnection: guolinke
      repositoryName: '$(Build.Repository.Name)'
      action: 'create'
      target: '$(Build.SourceVersion)'
      tagSource: 'auto'
      title: '$(Build.SourceBranchName)'
      assets: |
        $(Build.SourcesDirectory)/binaries/PackageAssets/*
446
        $(Build.SourcesDirectory)/R/R-package/*
447
448
        $(Build.ArtifactStagingDirectory)/nuget/*.nupkg
        $(Build.ArtifactStagingDirectory)/archives/*
449
450
451
      assetUploadMode: 'delete'
      isDraft: true
      isPreRelease: false
Guolin Ke's avatar
Guolin Ke committed
452
      addChangeLog: false