.vsts-ci.yml 13.7 KB
Newer Older
1
2
3
trigger:
  branches:
    include:
4
    - master
5
6
7
  tags:
    include:
    - v*
8
9
pr:
- master
10
variables:
11
  AZURE: 'true'
12
  PYTHON_VERSION: '3.10'
13
  CONDA_ENV: test-env
14
15
16
17
  runCodesignValidationInjection: false
  skipComponentGovernanceDetection: true
  DOTNET_CLI_TELEMETRY_OPTOUT: true
  DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
18
19
20
resources:
  containers:
  - container: ubuntu1404
21
    image: lightgbm/vsts-agent:ubuntu-14.04
22
  - container: ubuntu-latest
23
    image: 'ubuntu:20.04'
24
    options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
25
  - container: rbase
26
    image: wch1/r-debug
27
jobs:
Guolin Ke's avatar
Guolin Ke committed
28
###########################################
29
- job: Linux
Guolin Ke's avatar
Guolin Ke committed
30
###########################################
31
  variables:
32
    COMPILER: gcc
33
    SETUP_CONDA: 'false'
34
35
    OS_NAME: 'linux'
    PRODUCES_ARTIFACTS: 'true'
36
  pool: sh-ubuntu
37
  container: ubuntu1404
38
  strategy:
Guolin Ke's avatar
Guolin Ke committed
39
40
41
    matrix:
      regular:
        TASK: regular
42
        PYTHON_VERSION: '3.9'
Guolin Ke's avatar
Guolin Ke committed
43
44
      sdist:
        TASK: sdist
45
        PYTHON_VERSION: '3.7'
Guolin Ke's avatar
Guolin Ke committed
46
47
      bdist:
        TASK: bdist
48
        PYTHON_VERSION: '3.8'
49
50
51
52
53
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
54
        PYTHON_VERSION: '3.8'
55
56
      gpu_source:
        TASK: gpu
Guolin Ke's avatar
Guolin Ke committed
57
        METHOD: source
58
59
        # on Ubuntu 14.04, gpu_source build segfaults with newer version of Python
        # (and newer version of scipy as a consequence)
60
        PYTHON_VERSION: '3.7'
61
62
      swig:
        TASK: swig
Guolin Ke's avatar
Guolin Ke committed
63
  steps:
64
  - script: |
65
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
66
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
Nikita Titov's avatar
Nikita Titov committed
67
68
      echo "##vso[task.prependpath]$CONDA/bin"
      AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
69
70
71
72
73
      echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
      LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
      echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
      echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
    displayName: 'Set variables'
74
75
76
  - script: |
      echo '$(Build.SourceVersion)' > '$(Build.ArtifactStagingDirectory)/commit.txt'
    displayName: 'Add commit hash to artifacts archive'
77
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
78
    displayName: Setup
79
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
80
    displayName: Test
Nikita Titov's avatar
Nikita Titov committed
81
  - task: PublishBuildArtifacts@1
82
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Nikita Titov's avatar
Nikita Titov committed
83
84
85
86
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
87
###########################################
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
- job: Linux_latest
###########################################
  variables:
    COMPILER: clang
    DEBIAN_FRONTEND: 'noninteractive'
    IN_UBUNTU_LATEST_CONTAINER: 'true'
    OS_NAME: 'linux'
    SETUP_CONDA: 'true'
  pool: sh-ubuntu
  container: ubuntu-latest
  strategy:
    matrix:
      regular:
        TASK: regular
      sdist:
        TASK: sdist
      bdist:
        TASK: bdist
106
        PYTHON_VERSION: '3.8'
107
108
109
110
111
112
113
114
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
      mpi_pip:
        TASK: mpi
        METHOD: pip
115
        PYTHON_VERSION: '3.9'
116
117
118
      mpi_wheel:
        TASK: mpi
        METHOD: wheel
119
        PYTHON_VERSION: '3.7'
120
121
122
      gpu_source:
        TASK: gpu
        METHOD: source
123
        PYTHON_VERSION: '3.9'
124
125
126
      gpu_pip:
        TASK: gpu
        METHOD: pip
127
        PYTHON_VERSION: '3.8'
128
129
130
      gpu_wheel:
        TASK: gpu
        METHOD: wheel
131
        PYTHON_VERSION: '3.7'
132
133
134
      cpp_tests:
        TASK: cpp-tests
        METHOD: with-sanitizers
135
136
137
138
  steps:
  - script: |
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
139
      CONDA=$HOME/miniforge
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
      AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
      echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
      LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
      echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
      echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
    displayName: 'Set variables'
  # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
  - script: |
      /tmp/docker exec -t -u 0 ci-container \
      sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
    displayName: 'Install sudo'
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
    displayName: Setup
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
    displayName: Test
###########################################
158
159
160
161
162
163
164
165
- job: QEMU_multiarch
###########################################
  variables:
    COMPILER: gcc
    OS_NAME: 'linux'
    PRODUCES_ARTIFACTS: 'true'
  pool:
    vmImage: ubuntu-latest
166
  timeoutInMinutes: 180
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
  strategy:
    matrix:
      bdist:
        TASK: bdist
        ARCH: aarch64
  steps:
  - script: |
      sudo apt-get update
      sudo apt-get install --no-install-recommends -y \
        binfmt-support \
        qemu \
        qemu-user \
        qemu-user-static
    displayName: 'Install QEMU'
  - script: |
      docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
    displayName: 'Enable Docker multi-architecture support'
  - script: |
      export ROOT_DOCKER_FOLDER=/LightGBM
      cat > docker.env <<EOF
      AZURE=$AZURE
      OS_NAME=$OS_NAME
      COMPILER=$COMPILER
      TASK=$TASK
      METHOD=$METHOD
      CONDA_ENV=$CONDA_ENV
      PYTHON_VERSION=$PYTHON_VERSION
      BUILD_DIRECTORY=$ROOT_DOCKER_FOLDER
      LGB_VER=$(head -n 1 VERSION.txt)
      PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS
      BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY
      EOF
      cat > docker-script.sh <<EOF
200
      export CONDA=\$HOME/miniforge
201
202
203
204
      export PATH=\$CONDA/bin:\$PATH
      $ROOT_DOCKER_FOLDER/.ci/setup.sh || exit -1
      $ROOT_DOCKER_FOLDER/.ci/test.sh || exit -1
      EOF
205
206
      IMAGE_URI="quay.io/pypa/manylinux2014_${ARCH}"
      docker pull "${IMAGE_URI}" || exit -1
207
      PLATFORM=$(docker inspect --format='{{.Os}}/{{.Architecture}}' "${IMAGE_URI}") || exit -1
208
      echo "detected image platform: ${PLATFORM}"
209
      docker run \
210
        --platform "${PLATFORM}" \
211
212
213
214
        --rm \
        --env-file docker.env \
        -v "$(Build.SourcesDirectory)":"$ROOT_DOCKER_FOLDER" \
        -v "$(Build.ArtifactStagingDirectory)":"$(Build.ArtifactStagingDirectory)" \
215
        "${IMAGE_URI}" \
216
217
218
219
220
221
222
223
224
        /bin/bash $ROOT_DOCKER_FOLDER/docker-script.sh
    displayName: 'Setup and run tests'
  - task: PublishBuildArtifacts@1
    condition: and(succeeded(), in(variables['TASK'], 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
###########################################
225
- job: macOS
Guolin Ke's avatar
Guolin Ke committed
226
###########################################
227
  variables:
228
    COMPILER: clang
229
230
    OS_NAME: 'macos'
    PRODUCES_ARTIFACTS: 'true'
231
  pool:
232
    vmImage: 'macOS-10.15'
233
  strategy:
Guolin Ke's avatar
Guolin Ke committed
234
235
236
    matrix:
      regular:
        TASK: regular
237
        PYTHON_VERSION: '3.7'
Guolin Ke's avatar
Guolin Ke committed
238
239
      sdist:
        TASK: sdist
240
        PYTHON_VERSION: '3.8'
Guolin Ke's avatar
Guolin Ke committed
241
242
      bdist:
        TASK: bdist
243
244
      swig:
        TASK: swig
245
246
      cpp_tests:
        TASK: cpp-tests
247
248
        METHOD: with-sanitizers
        SANITIZERS: "address;undefined"
Guolin Ke's avatar
Guolin Ke committed
249
  steps:
250
  - script: |
251
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
252
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
253
      CONDA=$AGENT_HOMEDIRECTORY/miniforge
Nikita Titov's avatar
Nikita Titov committed
254
255
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
256
      echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64"
257
    displayName: 'Set variables'
258
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
259
    displayName: Setup
260
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
261
    displayName: Test
Guolin Ke's avatar
Guolin Ke committed
262
  - task: PublishBuildArtifacts@1
263
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
264
    inputs:
265
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
266
267
268
      artifactName: PackageAssets
      artifactType: container
###########################################
269
- job: Windows
Guolin Ke's avatar
Guolin Ke committed
270
###########################################
271
  pool:
272
    vmImage: 'windows-2019'
273
  strategy:
Guolin Ke's avatar
Guolin Ke committed
274
275
276
    matrix:
      regular:
        TASK: regular
277
        PYTHON_VERSION: '3.9'
Guolin Ke's avatar
Guolin Ke committed
278
279
      sdist:
        TASK: sdist
280
        PYTHON_VERSION: '3.8'
Guolin Ke's avatar
Guolin Ke committed
281
282
      bdist:
        TASK: bdist
283
284
      swig:
        TASK: swig
285
286
      cpp_tests:
        TASK: cpp-tests
Guolin Ke's avatar
Guolin Ke committed
287
  steps:
288
289
290
  - powershell: |
      Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
    displayName: 'Set Variables'
291
292
  - script: |
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install_opencl.ps1"
293
    condition: eq(variables['TASK'], 'bdist')
294
    displayName: 'Install OpenCL'
Nikita Titov's avatar
Nikita Titov committed
295
  - script: |
296
297
      cmd /c "conda config --add channels conda-forge"
      cmd /c "conda config --set channel_priority strict"
298
299
      cmd /c "conda init powershell"
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1"
300
    displayName: Test
Guolin Ke's avatar
Guolin Ke committed
301
  - task: PublishBuildArtifacts@1
302
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
303
    inputs:
304
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
305
306
      artifactName: PackageAssets
      artifactType: container
307
308
309
310
311
312
313
314
315
316
###########################################
- job: R_artifact
###########################################
  condition: not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
  pool:
    vmImage: 'ubuntu-latest'
  container: rbase
  steps:
  - script: |
      LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g")
317
318
319
      R_LIB_PATH=~/Rlib
      export R_LIBS=${R_LIB_PATH}
      mkdir -p ${R_LIB_PATH}
320
      RDscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'RhpcBLASctl', 'rmarkdown'),  lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.rstudio.com', Ncpus = parallel::detectCores())" || exit -1
321
      sh build-cran-package.sh --r-executable=RD || exit -1
322
323
324
325
326
327
328
329
      mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz
    displayName: 'Build CRAN R-package'
  - task: PublishBuildArtifacts@1
    condition: succeeded()
    inputs:
      pathtoPublish: $(Build.ArtifactStagingDirectory)
      artifactName: R-package
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
330

331
###########################################
332
- job: Package
333
###########################################
Guolin Ke's avatar
Guolin Ke committed
334
335
  dependsOn:
  - Linux
336
  - Linux_latest
337
  - QEMU_multiarch
338
  - macOS
Guolin Ke's avatar
Guolin Ke committed
339
  - Windows
340
  - R_artifact
341
  condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
342
  pool:
343
    vmImage: 'ubuntu-latest'
Guolin Ke's avatar
Guolin Ke committed
344
  steps:
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
  # Create archives with complete source code included (with git submodules)
  - task: ArchiveFiles@2
    displayName: Create zip archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: zip
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_zip.zip'
      replaceExistingArchive: true
  - task: ArchiveFiles@2
    displayName: Create tar.gz archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: tar
      tarCompression: gz
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_tar_gz.tar.gz'
      replaceExistingArchive: true
Guolin Ke's avatar
Guolin Ke committed
365
366
367
368
369
370
  # Download all agent packages from all previous phases
  - task: DownloadBuildArtifacts@0
    displayName: Download package assets
    inputs:
      artifactName: PackageAssets
      downloadPath: $(Build.SourcesDirectory)/binaries
371
372
373
374
375
376
  - task: DownloadBuildArtifacts@0
    displayName: Download R-package
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      artifactName: R-package
      downloadPath: $(Build.SourcesDirectory)/R
Guolin Ke's avatar
Guolin Ke committed
377
  - script: |
378
      python "$(Build.SourcesDirectory)/.nuget/create_nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
379
380
381
382
383
    displayName: 'Create NuGet configuration files'
  - task: NuGetCommand@2
    inputs:
      command: pack
      packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec'
384
      packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
Guolin Ke's avatar
Guolin Ke committed
385
386
  - task: PublishBuildArtifacts@1
    inputs:
387
      pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
388
      artifactName: NuGet
Guolin Ke's avatar
Guolin Ke committed
389
      artifactType: container
390
391
392
393
394
395
396
397
398
399
400
401
  - task: GitHubRelease@0
    displayName: 'Create GitHub Release'
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      gitHubConnection: guolinke
      repositoryName: '$(Build.Repository.Name)'
      action: 'create'
      target: '$(Build.SourceVersion)'
      tagSource: 'auto'
      title: '$(Build.SourceBranchName)'
      assets: |
        $(Build.SourcesDirectory)/binaries/PackageAssets/*
402
        $(Build.SourcesDirectory)/R/R-package/*
403
404
        $(Build.ArtifactStagingDirectory)/nuget/*.nupkg
        $(Build.ArtifactStagingDirectory)/archives/*
405
406
407
      assetUploadMode: 'delete'
      isDraft: true
      isPreRelease: false
Guolin Ke's avatar
Guolin Ke committed
408
      addChangeLog: false