.vsts-ci.yml 13.4 KB
Newer Older
1
2
3
trigger:
  branches:
    include:
4
    - master
5
6
7
  tags:
    include:
    - v*
8
9
pr:
- master
10
variables:
11
  AZURE: 'true'
12
  PYTHON_VERSION: '3.10'
13
  CONDA_ENV: test-env
14
15
16
resources:
  containers:
  - container: ubuntu1404
17
    image: lightgbm/vsts-agent:ubuntu-14.04
18
19
20
  - container: ubuntu-latest
    image: 'ubuntu:latest'
    options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
21
  - container: rbase
22
    image: wch1/r-debug
23
jobs:
Guolin Ke's avatar
Guolin Ke committed
24
###########################################
25
- job: Linux
Guolin Ke's avatar
Guolin Ke committed
26
###########################################
27
  variables:
28
    COMPILER: gcc
29
    SETUP_CONDA: 'false'
30
31
    OS_NAME: 'linux'
    PRODUCES_ARTIFACTS: 'true'
32
  pool: sh-ubuntu
33
  container: ubuntu1404
34
  strategy:
Guolin Ke's avatar
Guolin Ke committed
35
36
37
    matrix:
      regular:
        TASK: regular
38
        PYTHON_VERSION: '3.9'
Guolin Ke's avatar
Guolin Ke committed
39
40
      sdist:
        TASK: sdist
41
        PYTHON_VERSION: '3.7'
Guolin Ke's avatar
Guolin Ke committed
42
43
      bdist:
        TASK: bdist
44
        PYTHON_VERSION: '3.8'
45
46
47
48
49
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
50
        PYTHON_VERSION: '3.8'
51
52
      gpu_source:
        TASK: gpu
Guolin Ke's avatar
Guolin Ke committed
53
        METHOD: source
54
55
        # on Ubuntu 14.04, gpu_source build segfaults with newer version of Python
        # (and newer version of scipy as a consequence)
56
        PYTHON_VERSION: '3.7'
57
58
      swig:
        TASK: swig
Guolin Ke's avatar
Guolin Ke committed
59
  steps:
60
  - script: |
61
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
62
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
Nikita Titov's avatar
Nikita Titov committed
63
64
      echo "##vso[task.prependpath]$CONDA/bin"
      AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
65
66
67
68
69
      echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
      LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
      echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
      echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
    displayName: 'Set variables'
70
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
71
    displayName: Setup
72
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
73
    displayName: Test
Nikita Titov's avatar
Nikita Titov committed
74
  - task: PublishBuildArtifacts@1
75
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Nikita Titov's avatar
Nikita Titov committed
76
77
78
79
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
80
###########################################
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
- job: Linux_latest
###########################################
  variables:
    COMPILER: clang
    DEBIAN_FRONTEND: 'noninteractive'
    IN_UBUNTU_LATEST_CONTAINER: 'true'
    OS_NAME: 'linux'
    SETUP_CONDA: 'true'
  pool: sh-ubuntu
  container: ubuntu-latest
  strategy:
    matrix:
      regular:
        TASK: regular
      sdist:
        TASK: sdist
      bdist:
        TASK: bdist
99
        PYTHON_VERSION: '3.8'
100
101
102
103
104
105
106
107
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
      mpi_pip:
        TASK: mpi
        METHOD: pip
108
        PYTHON_VERSION: '3.9'
109
110
111
      mpi_wheel:
        TASK: mpi
        METHOD: wheel
112
        PYTHON_VERSION: '3.7'
113
114
115
      gpu_source:
        TASK: gpu
        METHOD: source
116
        PYTHON_VERSION: '3.9'
117
118
119
      gpu_pip:
        TASK: gpu
        METHOD: pip
120
        PYTHON_VERSION: '3.8'
121
122
123
      gpu_wheel:
        TASK: gpu
        METHOD: wheel
124
        PYTHON_VERSION: '3.7'
125
126
127
      cpp_tests:
        TASK: cpp-tests
        METHOD: with-sanitizers
128
129
130
131
  steps:
  - script: |
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
132
      CONDA=$HOME/miniforge
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
      AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
      echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
      LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
      echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
      echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
    displayName: 'Set variables'
  # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
  - script: |
      /tmp/docker exec -t -u 0 ci-container \
      sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
    displayName: 'Install sudo'
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
    displayName: Setup
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
    displayName: Test
###########################################
151
152
153
154
155
156
157
158
- job: QEMU_multiarch
###########################################
  variables:
    COMPILER: gcc
    OS_NAME: 'linux'
    PRODUCES_ARTIFACTS: 'true'
  pool:
    vmImage: ubuntu-latest
159
  timeoutInMinutes: 180
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
  strategy:
    matrix:
      bdist:
        TASK: bdist
        ARCH: aarch64
  steps:
  - script: |
      sudo apt-get update
      sudo apt-get install --no-install-recommends -y \
        binfmt-support \
        qemu \
        qemu-user \
        qemu-user-static
    displayName: 'Install QEMU'
  - script: |
      docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
    displayName: 'Enable Docker multi-architecture support'
  - script: |
      export ROOT_DOCKER_FOLDER=/LightGBM
      cat > docker.env <<EOF
      AZURE=$AZURE
      OS_NAME=$OS_NAME
      COMPILER=$COMPILER
      TASK=$TASK
      METHOD=$METHOD
      CONDA_ENV=$CONDA_ENV
      PYTHON_VERSION=$PYTHON_VERSION
      BUILD_DIRECTORY=$ROOT_DOCKER_FOLDER
      LGB_VER=$(head -n 1 VERSION.txt)
      PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS
      BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY
      EOF
      cat > docker-script.sh <<EOF
193
      export CONDA=\$HOME/miniforge
194
195
196
197
      export PATH=\$CONDA/bin:\$PATH
      $ROOT_DOCKER_FOLDER/.ci/setup.sh || exit -1
      $ROOT_DOCKER_FOLDER/.ci/test.sh || exit -1
      EOF
198
199
      IMAGE_URI="quay.io/pypa/manylinux2014_${ARCH}"
      docker pull "${IMAGE_URI}" || exit -1
200
      PLATFORM=$(docker inspect --format='{{.Os}}/{{.Architecture}}' "${IMAGE_URI}") || exit -1
201
      echo "detected image platform: ${PLATFORM}"
202
      docker run \
203
        --platform "${PLATFORM}" \
204
205
206
207
        --rm \
        --env-file docker.env \
        -v "$(Build.SourcesDirectory)":"$ROOT_DOCKER_FOLDER" \
        -v "$(Build.ArtifactStagingDirectory)":"$(Build.ArtifactStagingDirectory)" \
208
        "${IMAGE_URI}" \
209
210
211
212
213
214
215
216
217
        /bin/bash $ROOT_DOCKER_FOLDER/docker-script.sh
    displayName: 'Setup and run tests'
  - task: PublishBuildArtifacts@1
    condition: and(succeeded(), in(variables['TASK'], 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
###########################################
218
- job: macOS
Guolin Ke's avatar
Guolin Ke committed
219
###########################################
220
  variables:
221
    COMPILER: clang
222
223
    OS_NAME: 'macos'
    PRODUCES_ARTIFACTS: 'true'
224
  pool:
225
    vmImage: 'macOS-10.15'
226
  strategy:
Guolin Ke's avatar
Guolin Ke committed
227
228
229
    matrix:
      regular:
        TASK: regular
230
        PYTHON_VERSION: '3.7'
Guolin Ke's avatar
Guolin Ke committed
231
232
      sdist:
        TASK: sdist
233
        PYTHON_VERSION: '3.8'
Guolin Ke's avatar
Guolin Ke committed
234
235
      bdist:
        TASK: bdist
236
237
      swig:
        TASK: swig
238
239
      cpp_tests:
        TASK: cpp-tests
240
241
        METHOD: with-sanitizers
        SANITIZERS: "address;undefined"
Guolin Ke's avatar
Guolin Ke committed
242
  steps:
243
  - script: |
244
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
245
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
246
      CONDA=$AGENT_HOMEDIRECTORY/miniforge
Nikita Titov's avatar
Nikita Titov committed
247
248
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
249
      echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64"
250
    displayName: 'Set variables'
251
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
252
    displayName: Setup
253
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
254
    displayName: Test
Guolin Ke's avatar
Guolin Ke committed
255
  - task: PublishBuildArtifacts@1
256
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
257
    inputs:
258
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
259
260
261
      artifactName: PackageAssets
      artifactType: container
###########################################
262
- job: Windows
Guolin Ke's avatar
Guolin Ke committed
263
###########################################
264
265
266
  pool:
    vmImage: 'vs2017-win2016'
  strategy:
Guolin Ke's avatar
Guolin Ke committed
267
268
269
    matrix:
      regular:
        TASK: regular
270
        PYTHON_VERSION: '3.9'
Guolin Ke's avatar
Guolin Ke committed
271
272
      sdist:
        TASK: sdist
273
        PYTHON_VERSION: '3.8'
Guolin Ke's avatar
Guolin Ke committed
274
275
      bdist:
        TASK: bdist
276
277
      swig:
        TASK: swig
278
279
      cpp_tests:
        TASK: cpp-tests
Guolin Ke's avatar
Guolin Ke committed
280
  steps:
281
282
283
  - powershell: |
      Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
    displayName: 'Set Variables'
284
285
  - script: |
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install_opencl.ps1"
286
    condition: eq(variables['TASK'], 'bdist')
287
    displayName: 'Install OpenCL'
Nikita Titov's avatar
Nikita Titov committed
288
  - script: |
289
290
      cmd /c "conda config --add channels conda-forge"
      cmd /c "conda config --set channel_priority strict"
291
292
      cmd /c "conda init powershell"
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1"
293
    displayName: Test
Guolin Ke's avatar
Guolin Ke committed
294
  - task: PublishBuildArtifacts@1
295
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
296
    inputs:
297
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
298
299
      artifactName: PackageAssets
      artifactType: container
300
301
302
303
304
305
306
307
308
309
###########################################
- job: R_artifact
###########################################
  condition: not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
  pool:
    vmImage: 'ubuntu-latest'
  container: rbase
  steps:
  - script: |
      LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g")
310
311
312
313
314
      R_LIB_PATH=~/Rlib
      export R_LIBS=${R_LIB_PATH}
      mkdir -p ${R_LIB_PATH}
      RDscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'rmarkdown'),  lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" || exit -1
      sh build-cran-package.sh --r-executable=RD || exit -1
315
316
317
318
319
320
321
322
      mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz
    displayName: 'Build CRAN R-package'
  - task: PublishBuildArtifacts@1
    condition: succeeded()
    inputs:
      pathtoPublish: $(Build.ArtifactStagingDirectory)
      artifactName: R-package
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
323

324
###########################################
325
- job: Package
326
###########################################
Guolin Ke's avatar
Guolin Ke committed
327
328
  dependsOn:
  - Linux
329
  - Linux_latest
330
  - QEMU_multiarch
331
  - macOS
Guolin Ke's avatar
Guolin Ke committed
332
  - Windows
333
  - R_artifact
334
  condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
335
  pool:
336
    vmImage: 'ubuntu-latest'
Guolin Ke's avatar
Guolin Ke committed
337
  steps:
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
  # Create archives with complete source code included (with git submodules)
  - task: ArchiveFiles@2
    displayName: Create zip archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: zip
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_zip.zip'
      replaceExistingArchive: true
  - task: ArchiveFiles@2
    displayName: Create tar.gz archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: tar
      tarCompression: gz
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_tar_gz.tar.gz'
      replaceExistingArchive: true
Guolin Ke's avatar
Guolin Ke committed
358
359
360
361
362
363
  # Download all agent packages from all previous phases
  - task: DownloadBuildArtifacts@0
    displayName: Download package assets
    inputs:
      artifactName: PackageAssets
      downloadPath: $(Build.SourcesDirectory)/binaries
364
365
366
367
368
369
  - task: DownloadBuildArtifacts@0
    displayName: Download R-package
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      artifactName: R-package
      downloadPath: $(Build.SourcesDirectory)/R
Guolin Ke's avatar
Guolin Ke committed
370
  - script: |
371
      python "$(Build.SourcesDirectory)/.nuget/create_nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
372
373
374
375
376
    displayName: 'Create NuGet configuration files'
  - task: NuGetCommand@2
    inputs:
      command: pack
      packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec'
377
      packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
Guolin Ke's avatar
Guolin Ke committed
378
379
  - task: PublishBuildArtifacts@1
    inputs:
380
      pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
381
      artifactName: NuGet
Guolin Ke's avatar
Guolin Ke committed
382
      artifactType: container
383
384
385
386
387
388
389
390
391
392
393
394
  - task: GitHubRelease@0
    displayName: 'Create GitHub Release'
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      gitHubConnection: guolinke
      repositoryName: '$(Build.Repository.Name)'
      action: 'create'
      target: '$(Build.SourceVersion)'
      tagSource: 'auto'
      title: '$(Build.SourceBranchName)'
      assets: |
        $(Build.SourcesDirectory)/binaries/PackageAssets/*
395
        $(Build.SourcesDirectory)/R/R-package/*
396
397
        $(Build.ArtifactStagingDirectory)/nuget/*.nupkg
        $(Build.ArtifactStagingDirectory)/archives/*
398
399
400
      assetUploadMode: 'delete'
      isDraft: true
      isPreRelease: false
Guolin Ke's avatar
Guolin Ke committed
401
      addChangeLog: false