.vsts-ci.yml 13.4 KB
Newer Older
1
2
3
trigger:
  branches:
    include:
4
    - master
5
6
7
  tags:
    include:
    - v*
8
9
pr:
- master
10
variables:
11
  AZURE: 'true'
12
  PYTHON_VERSION: '3.10'
13
  CONDA_ENV: test-env
14
15
16
resources:
  containers:
  - container: ubuntu1404
17
    image: lightgbm/vsts-agent:ubuntu-14.04
18
19
20
  - container: ubuntu-latest
    image: 'ubuntu:latest'
    options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
21
  - container: rbase
22
    image: wch1/r-debug
23
jobs:
Guolin Ke's avatar
Guolin Ke committed
24
###########################################
25
- job: Linux
Guolin Ke's avatar
Guolin Ke committed
26
###########################################
27
  variables:
28
    COMPILER: gcc
29
    SETUP_CONDA: 'false'
30
31
    OS_NAME: 'linux'
    PRODUCES_ARTIFACTS: 'true'
32
  pool: sh-ubuntu
33
  container: ubuntu1404
34
  strategy:
Guolin Ke's avatar
Guolin Ke committed
35
36
37
    matrix:
      regular:
        TASK: regular
38
        PYTHON_VERSION: '3.9'
Guolin Ke's avatar
Guolin Ke committed
39
40
      sdist:
        TASK: sdist
41
        PYTHON_VERSION: '3.7'
Guolin Ke's avatar
Guolin Ke committed
42
43
      bdist:
        TASK: bdist
44
45
46
47
48
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
49
        PYTHON_VERSION: '3.8'
50
51
      gpu_source:
        TASK: gpu
Guolin Ke's avatar
Guolin Ke committed
52
        METHOD: source
53
54
55
        # on Ubuntu 14.04, gpu_source build segfaults with newer version of Python
        # (and newer version of scipy as a consequence)
        PYTHON_VERSION: 3.7
56
57
      swig:
        TASK: swig
Guolin Ke's avatar
Guolin Ke committed
58
  steps:
59
  - script: |
60
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
61
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
Nikita Titov's avatar
Nikita Titov committed
62
63
      echo "##vso[task.prependpath]$CONDA/bin"
      AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
64
65
66
67
68
      echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
      LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
      echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
      echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
    displayName: 'Set variables'
69
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
70
    displayName: Setup
71
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
72
    displayName: Test
Nikita Titov's avatar
Nikita Titov committed
73
  - task: PublishBuildArtifacts@1
74
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Nikita Titov's avatar
Nikita Titov committed
75
76
77
78
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
79
###########################################
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
- job: Linux_latest
###########################################
  variables:
    COMPILER: clang
    DEBIAN_FRONTEND: 'noninteractive'
    IN_UBUNTU_LATEST_CONTAINER: 'true'
    OS_NAME: 'linux'
    SETUP_CONDA: 'true'
  pool: sh-ubuntu
  container: ubuntu-latest
  strategy:
    matrix:
      regular:
        TASK: regular
      sdist:
        TASK: sdist
      bdist:
        TASK: bdist
98
        PYTHON_VERSION: '3.8'
99
100
101
102
103
104
105
106
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
      mpi_pip:
        TASK: mpi
        METHOD: pip
107
        PYTHON_VERSION: '3.9'
108
109
110
      mpi_wheel:
        TASK: mpi
        METHOD: wheel
111
        PYTHON_VERSION: '3.7'
112
113
114
      gpu_source:
        TASK: gpu
        METHOD: source
115
        PYTHON_VERSION: '3.9'
116
117
118
      gpu_pip:
        TASK: gpu
        METHOD: pip
119
        PYTHON_VERSION: '3.8'
120
121
122
      gpu_wheel:
        TASK: gpu
        METHOD: wheel
123
        PYTHON_VERSION: '3.7'
124
125
126
      cpp_tests:
        TASK: cpp-tests
        METHOD: with-sanitizers
127
128
129
130
  steps:
  - script: |
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
131
      CONDA=$HOME/miniforge
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
      AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
      echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
      LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
      echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
      echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
    displayName: 'Set variables'
  # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
  - script: |
      /tmp/docker exec -t -u 0 ci-container \
      sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
    displayName: 'Install sudo'
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
    displayName: Setup
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
    displayName: Test
###########################################
150
151
152
153
154
155
156
157
- job: QEMU_multiarch
###########################################
  variables:
    COMPILER: gcc
    OS_NAME: 'linux'
    PRODUCES_ARTIFACTS: 'true'
  pool:
    vmImage: ubuntu-latest
158
  timeoutInMinutes: 180
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
  strategy:
    matrix:
      bdist:
        TASK: bdist
        ARCH: aarch64
  steps:
  - script: |
      sudo apt-get update
      sudo apt-get install --no-install-recommends -y \
        binfmt-support \
        qemu \
        qemu-user \
        qemu-user-static
    displayName: 'Install QEMU'
  - script: |
      docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
    displayName: 'Enable Docker multi-architecture support'
  - script: |
      export ROOT_DOCKER_FOLDER=/LightGBM
      cat > docker.env <<EOF
      AZURE=$AZURE
      OS_NAME=$OS_NAME
      COMPILER=$COMPILER
      TASK=$TASK
      METHOD=$METHOD
      CONDA_ENV=$CONDA_ENV
      PYTHON_VERSION=$PYTHON_VERSION
      BUILD_DIRECTORY=$ROOT_DOCKER_FOLDER
      LGB_VER=$(head -n 1 VERSION.txt)
      PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS
      BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY
      EOF
      cat > docker-script.sh <<EOF
192
      export CONDA=\$HOME/miniforge
193
194
195
196
      export PATH=\$CONDA/bin:\$PATH
      $ROOT_DOCKER_FOLDER/.ci/setup.sh || exit -1
      $ROOT_DOCKER_FOLDER/.ci/test.sh || exit -1
      EOF
197
198
      IMAGE_URI="quay.io/pypa/manylinux2014_${ARCH}"
      docker pull "${IMAGE_URI}" || exit -1
199
      PLATFORM=$(docker inspect --format='{{.Os}}/{{.Architecture}}' "${IMAGE_URI}") || exit -1
200
      echo "detected image platform: ${PLATFORM}"
201
      docker run \
202
        --platform "${PLATFORM}" \
203
204
205
206
        --rm \
        --env-file docker.env \
        -v "$(Build.SourcesDirectory)":"$ROOT_DOCKER_FOLDER" \
        -v "$(Build.ArtifactStagingDirectory)":"$(Build.ArtifactStagingDirectory)" \
207
        "${IMAGE_URI}" \
208
209
210
211
212
213
214
215
216
        /bin/bash $ROOT_DOCKER_FOLDER/docker-script.sh
    displayName: 'Setup and run tests'
  - task: PublishBuildArtifacts@1
    condition: and(succeeded(), in(variables['TASK'], 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
###########################################
217
- job: macOS
Guolin Ke's avatar
Guolin Ke committed
218
###########################################
219
  variables:
220
    COMPILER: clang
221
222
    OS_NAME: 'macos'
    PRODUCES_ARTIFACTS: 'true'
223
  pool:
224
    vmImage: 'macOS-10.15'
225
  strategy:
Guolin Ke's avatar
Guolin Ke committed
226
227
228
    matrix:
      regular:
        TASK: regular
229
        PYTHON_VERSION: '3.7'
Guolin Ke's avatar
Guolin Ke committed
230
231
      sdist:
        TASK: sdist
232
        PYTHON_VERSION: '3.8'
Guolin Ke's avatar
Guolin Ke committed
233
234
      bdist:
        TASK: bdist
235
236
      swig:
        TASK: swig
237
238
      cpp_tests:
        TASK: cpp-tests
239
240
        METHOD: with-sanitizers
        SANITIZERS: "address;undefined"
Guolin Ke's avatar
Guolin Ke committed
241
  steps:
242
  - script: |
243
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
244
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
245
      CONDA=$AGENT_HOMEDIRECTORY/miniforge
Nikita Titov's avatar
Nikita Titov committed
246
247
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
248
      echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64"
249
    displayName: 'Set variables'
250
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
251
    displayName: Setup
252
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
253
    displayName: Test
Guolin Ke's avatar
Guolin Ke committed
254
  - task: PublishBuildArtifacts@1
255
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
256
    inputs:
257
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
258
259
260
      artifactName: PackageAssets
      artifactType: container
###########################################
261
- job: Windows
Guolin Ke's avatar
Guolin Ke committed
262
###########################################
263
264
265
  pool:
    vmImage: 'vs2017-win2016'
  strategy:
Guolin Ke's avatar
Guolin Ke committed
266
267
268
    matrix:
      regular:
        TASK: regular
269
        PYTHON_VERSION: '3.9'
Guolin Ke's avatar
Guolin Ke committed
270
271
      sdist:
        TASK: sdist
272
        PYTHON_VERSION: '3.8'
Guolin Ke's avatar
Guolin Ke committed
273
274
      bdist:
        TASK: bdist
275
276
      swig:
        TASK: swig
277
278
      cpp_tests:
        TASK: cpp-tests
Guolin Ke's avatar
Guolin Ke committed
279
  steps:
280
281
282
  - powershell: |
      Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
    displayName: 'Set Variables'
283
284
  - script: |
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install_opencl.ps1"
285
    condition: eq(variables['TASK'], 'bdist')
286
    displayName: 'Install OpenCL'
Nikita Titov's avatar
Nikita Titov committed
287
  - script: |
288
289
      cmd /c "conda config --add channels conda-forge"
      cmd /c "conda config --set channel_priority strict"
290
291
      cmd /c "conda init powershell"
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1"
292
    displayName: Test
Guolin Ke's avatar
Guolin Ke committed
293
  - task: PublishBuildArtifacts@1
294
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
295
    inputs:
296
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
297
298
      artifactName: PackageAssets
      artifactType: container
299
300
301
302
303
304
305
306
307
308
###########################################
- job: R_artifact
###########################################
  condition: not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
  pool:
    vmImage: 'ubuntu-latest'
  container: rbase
  steps:
  - script: |
      LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g")
309
310
311
312
313
      R_LIB_PATH=~/Rlib
      export R_LIBS=${R_LIB_PATH}
      mkdir -p ${R_LIB_PATH}
      RDscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'rmarkdown'),  lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" || exit -1
      sh build-cran-package.sh --r-executable=RD || exit -1
314
315
316
317
318
319
320
321
      mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz
    displayName: 'Build CRAN R-package'
  - task: PublishBuildArtifacts@1
    condition: succeeded()
    inputs:
      pathtoPublish: $(Build.ArtifactStagingDirectory)
      artifactName: R-package
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
322

323
###########################################
324
- job: Package
325
###########################################
Guolin Ke's avatar
Guolin Ke committed
326
327
  dependsOn:
  - Linux
328
  - Linux_latest
329
  - QEMU_multiarch
330
  - macOS
Guolin Ke's avatar
Guolin Ke committed
331
  - Windows
332
  - R_artifact
333
  condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
334
  pool:
335
    vmImage: 'ubuntu-latest'
Guolin Ke's avatar
Guolin Ke committed
336
  steps:
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
  # Create archives with complete source code included (with git submodules)
  - task: ArchiveFiles@2
    displayName: Create zip archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: zip
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_zip.zip'
      replaceExistingArchive: true
  - task: ArchiveFiles@2
    displayName: Create tar.gz archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: tar
      tarCompression: gz
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_tar_gz.tar.gz'
      replaceExistingArchive: true
Guolin Ke's avatar
Guolin Ke committed
357
358
359
360
361
362
  # Download all agent packages from all previous phases
  - task: DownloadBuildArtifacts@0
    displayName: Download package assets
    inputs:
      artifactName: PackageAssets
      downloadPath: $(Build.SourcesDirectory)/binaries
363
364
365
366
367
368
  - task: DownloadBuildArtifacts@0
    displayName: Download R-package
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      artifactName: R-package
      downloadPath: $(Build.SourcesDirectory)/R
Guolin Ke's avatar
Guolin Ke committed
369
  - script: |
370
      python "$(Build.SourcesDirectory)/.nuget/create_nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
371
372
373
374
375
    displayName: 'Create NuGet configuration files'
  - task: NuGetCommand@2
    inputs:
      command: pack
      packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec'
376
      packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
Guolin Ke's avatar
Guolin Ke committed
377
378
  - task: PublishBuildArtifacts@1
    inputs:
379
      pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
380
      artifactName: NuGet
Guolin Ke's avatar
Guolin Ke committed
381
      artifactType: container
382
383
384
385
386
387
388
389
390
391
392
393
  - task: GitHubRelease@0
    displayName: 'Create GitHub Release'
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      gitHubConnection: guolinke
      repositoryName: '$(Build.Repository.Name)'
      action: 'create'
      target: '$(Build.SourceVersion)'
      tagSource: 'auto'
      title: '$(Build.SourceBranchName)'
      assets: |
        $(Build.SourcesDirectory)/binaries/PackageAssets/*
394
        $(Build.SourcesDirectory)/R/R-package/*
395
396
        $(Build.ArtifactStagingDirectory)/nuget/*.nupkg
        $(Build.ArtifactStagingDirectory)/archives/*
397
398
399
      assetUploadMode: 'delete'
      isDraft: true
      isPreRelease: false
Guolin Ke's avatar
Guolin Ke committed
400
      addChangeLog: false