.vsts-ci.yml 13.2 KB
Newer Older
1
2
3
trigger:
  branches:
    include:
4
    - master
5
6
7
  tags:
    include:
    - v*
8
9
pr:
- master
10
variables:
11
  AZURE: 'true'
12
  PYTHON_VERSION: 3.9
13
  CONDA_ENV: test-env
14
15
16
resources:
  containers:
  - container: ubuntu1404
17
    image: lightgbm/vsts-agent:ubuntu-14.04
18
19
20
  - container: ubuntu-latest
    image: 'ubuntu:latest'
    options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
21
  - container: rbase
22
    image: wch1/r-debug
23
jobs:
Guolin Ke's avatar
Guolin Ke committed
24
###########################################
25
- job: Linux
Guolin Ke's avatar
Guolin Ke committed
26
###########################################
27
  variables:
28
    COMPILER: gcc
29
    SETUP_CONDA: 'false'
30
31
    OS_NAME: 'linux'
    PRODUCES_ARTIFACTS: 'true'
32
  pool: sh-ubuntu
33
  container: ubuntu1404
34
  strategy:
Guolin Ke's avatar
Guolin Ke committed
35
36
37
38
39
    matrix:
      regular:
        TASK: regular
      sdist:
        TASK: sdist
40
        PYTHON_VERSION: 3.7
Guolin Ke's avatar
Guolin Ke committed
41
42
      bdist:
        TASK: bdist
43
44
45
46
47
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
48
        PYTHON_VERSION: 3.8
49
50
      gpu_source:
        TASK: gpu
Guolin Ke's avatar
Guolin Ke committed
51
        METHOD: source
52
53
54
        # on Ubuntu 14.04, gpu_source build segfaults with newer version of Python
        # (and newer version of scipy as a consequence)
        PYTHON_VERSION: 3.7
55
56
      swig:
        TASK: swig
Guolin Ke's avatar
Guolin Ke committed
57
  steps:
58
  - script: |
59
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
60
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
Nikita Titov's avatar
Nikita Titov committed
61
62
      echo "##vso[task.prependpath]$CONDA/bin"
      AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
63
64
65
66
67
      echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
      LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
      echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
      echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
    displayName: 'Set variables'
68
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
69
    displayName: Setup
70
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
71
    displayName: Test
Nikita Titov's avatar
Nikita Titov committed
72
  - task: PublishBuildArtifacts@1
73
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Nikita Titov's avatar
Nikita Titov committed
74
75
76
77
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
78
###########################################
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
- job: Linux_latest
###########################################
  variables:
    COMPILER: clang
    DEBIAN_FRONTEND: 'noninteractive'
    IN_UBUNTU_LATEST_CONTAINER: 'true'
    OS_NAME: 'linux'
    SETUP_CONDA: 'true'
  pool: sh-ubuntu
  container: ubuntu-latest
  strategy:
    matrix:
      regular:
        TASK: regular
      sdist:
        TASK: sdist
      bdist:
        TASK: bdist
97
        PYTHON_VERSION: 3.8
98
99
100
101
102
103
104
105
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
      mpi_pip:
        TASK: mpi
        METHOD: pip
106
        PYTHON_VERSION: 3.8
107
108
109
110
111
112
113
114
115
116
117
118
119
120
      mpi_wheel:
        TASK: mpi
        METHOD: wheel
        PYTHON_VERSION: 3.7
      gpu_source:
        TASK: gpu
        METHOD: source
      gpu_pip:
        TASK: gpu
        METHOD: pip
      gpu_wheel:
        TASK: gpu
        METHOD: wheel
        PYTHON_VERSION: 3.7
121
122
123
      cpp_tests:
        TASK: cpp-tests
        METHOD: with-sanitizers
124
125
126
127
  steps:
  - script: |
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
128
      CONDA=$HOME/miniforge
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
      AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
      echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
      LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
      echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
      echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
    displayName: 'Set variables'
  # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
  - script: |
      /tmp/docker exec -t -u 0 ci-container \
      sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
    displayName: 'Install sudo'
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
    displayName: Setup
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
    displayName: Test
###########################################
147
148
149
150
151
152
153
154
- job: QEMU_multiarch
###########################################
  variables:
    COMPILER: gcc
    OS_NAME: 'linux'
    PRODUCES_ARTIFACTS: 'true'
  pool:
    vmImage: ubuntu-latest
155
  timeoutInMinutes: 180
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
  strategy:
    matrix:
      bdist:
        TASK: bdist
        ARCH: aarch64
  steps:
  - script: |
      sudo apt-get update
      sudo apt-get install --no-install-recommends -y \
        binfmt-support \
        qemu \
        qemu-user \
        qemu-user-static
    displayName: 'Install QEMU'
  - script: |
      docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
    displayName: 'Enable Docker multi-architecture support'
  - script: |
      export ROOT_DOCKER_FOLDER=/LightGBM
      cat > docker.env <<EOF
      AZURE=$AZURE
      OS_NAME=$OS_NAME
      COMPILER=$COMPILER
      TASK=$TASK
      METHOD=$METHOD
      CONDA_ENV=$CONDA_ENV
      PYTHON_VERSION=$PYTHON_VERSION
      BUILD_DIRECTORY=$ROOT_DOCKER_FOLDER
      LGB_VER=$(head -n 1 VERSION.txt)
      PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS
      BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY
      EOF
      cat > docker-script.sh <<EOF
189
      export CONDA=\$HOME/miniforge
190
191
192
193
      export PATH=\$CONDA/bin:\$PATH
      $ROOT_DOCKER_FOLDER/.ci/setup.sh || exit -1
      $ROOT_DOCKER_FOLDER/.ci/test.sh || exit -1
      EOF
194
195
      IMAGE_URI="quay.io/pypa/manylinux2014_${ARCH}"
      docker pull "${IMAGE_URI}" || exit -1
196
      PLATFORM=$(docker inspect --format='{{.Os}}/{{.Architecture}}' "${IMAGE_URI}") || exit -1
197
      echo "detected image platform: ${PLATFORM}"
198
      docker run \
199
        --platform "${PLATFORM}" \
200
201
202
203
        --rm \
        --env-file docker.env \
        -v "$(Build.SourcesDirectory)":"$ROOT_DOCKER_FOLDER" \
        -v "$(Build.ArtifactStagingDirectory)":"$(Build.ArtifactStagingDirectory)" \
204
        "${IMAGE_URI}" \
205
206
207
208
209
210
211
212
213
        /bin/bash $ROOT_DOCKER_FOLDER/docker-script.sh
    displayName: 'Setup and run tests'
  - task: PublishBuildArtifacts@1
    condition: and(succeeded(), in(variables['TASK'], 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
###########################################
214
- job: macOS
Guolin Ke's avatar
Guolin Ke committed
215
###########################################
216
  variables:
217
    COMPILER: clang
218
219
    OS_NAME: 'macos'
    PRODUCES_ARTIFACTS: 'true'
220
  pool:
221
    vmImage: 'macOS-10.15'
222
  strategy:
Guolin Ke's avatar
Guolin Ke committed
223
224
225
    matrix:
      regular:
        TASK: regular
226
        PYTHON_VERSION: 3.7
Guolin Ke's avatar
Guolin Ke committed
227
228
      sdist:
        TASK: sdist
229
        PYTHON_VERSION: 3.8
Guolin Ke's avatar
Guolin Ke committed
230
231
      bdist:
        TASK: bdist
232
233
      swig:
        TASK: swig
234
235
      cpp_tests:
        TASK: cpp-tests
236
237
        METHOD: with-sanitizers
        SANITIZERS: "address;undefined"
Guolin Ke's avatar
Guolin Ke committed
238
  steps:
239
  - script: |
240
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
241
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
242
      CONDA=$AGENT_HOMEDIRECTORY/miniforge
Nikita Titov's avatar
Nikita Titov committed
243
244
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
245
      echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64"
246
    displayName: 'Set variables'
247
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
248
    displayName: Setup
249
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
250
    displayName: Test
Guolin Ke's avatar
Guolin Ke committed
251
  - task: PublishBuildArtifacts@1
252
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
253
    inputs:
254
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
255
256
257
      artifactName: PackageAssets
      artifactType: container
###########################################
258
- job: Windows
Guolin Ke's avatar
Guolin Ke committed
259
###########################################
260
261
262
  pool:
    vmImage: 'vs2017-win2016'
  strategy:
Guolin Ke's avatar
Guolin Ke committed
263
264
265
266
267
    matrix:
      regular:
        TASK: regular
      sdist:
        TASK: sdist
268
        PYTHON_VERSION: 3.8
Guolin Ke's avatar
Guolin Ke committed
269
270
      bdist:
        TASK: bdist
271
272
      swig:
        TASK: swig
273
274
      cpp_tests:
        TASK: cpp-tests
Guolin Ke's avatar
Guolin Ke committed
275
  steps:
276
277
278
  - powershell: |
      Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
    displayName: 'Set Variables'
279
280
  - script: |
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install_opencl.ps1"
281
    condition: eq(variables['TASK'], 'bdist')
282
    displayName: 'Install OpenCL'
Nikita Titov's avatar
Nikita Titov committed
283
  - script: |
284
285
      cmd /c "conda config --add channels conda-forge"
      cmd /c "conda config --set channel_priority strict"
286
287
      cmd /c "conda init powershell"
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1"
288
    displayName: Test
Guolin Ke's avatar
Guolin Ke committed
289
  - task: PublishBuildArtifacts@1
290
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
291
    inputs:
292
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
293
294
      artifactName: PackageAssets
      artifactType: container
295
296
297
298
299
300
301
302
303
304
###########################################
- job: R_artifact
###########################################
  condition: not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
  pool:
    vmImage: 'ubuntu-latest'
  container: rbase
  steps:
  - script: |
      LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g")
305
306
307
308
309
      R_LIB_PATH=~/Rlib
      export R_LIBS=${R_LIB_PATH}
      mkdir -p ${R_LIB_PATH}
      RDscript -e "install.packages(c('R6', 'data.table', 'jsonlite', 'knitr', 'Matrix', 'rmarkdown'),  lib = '${R_LIB_PATH}', dependencies = c('Depends', 'Imports', 'LinkingTo'), repos = 'https://cran.r-project.org', Ncpus = parallel::detectCores())" || exit -1
      sh build-cran-package.sh --r-executable=RD || exit -1
310
311
312
313
314
315
316
317
      mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz
    displayName: 'Build CRAN R-package'
  - task: PublishBuildArtifacts@1
    condition: succeeded()
    inputs:
      pathtoPublish: $(Build.ArtifactStagingDirectory)
      artifactName: R-package
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
318

319
###########################################
320
- job: Package
321
###########################################
Guolin Ke's avatar
Guolin Ke committed
322
323
  dependsOn:
  - Linux
324
  - Linux_latest
325
  - QEMU_multiarch
326
  - macOS
Guolin Ke's avatar
Guolin Ke committed
327
  - Windows
328
  - R_artifact
329
  condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
330
  pool:
331
    vmImage: 'ubuntu-latest'
Guolin Ke's avatar
Guolin Ke committed
332
  steps:
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
  # Create archives with complete source code included (with git submodules)
  - task: ArchiveFiles@2
    displayName: Create zip archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: zip
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_zip.zip'
      replaceExistingArchive: true
  - task: ArchiveFiles@2
    displayName: Create tar.gz archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: tar
      tarCompression: gz
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_tar_gz.tar.gz'
      replaceExistingArchive: true
Guolin Ke's avatar
Guolin Ke committed
353
354
355
356
357
358
  # Download all agent packages from all previous phases
  - task: DownloadBuildArtifacts@0
    displayName: Download package assets
    inputs:
      artifactName: PackageAssets
      downloadPath: $(Build.SourcesDirectory)/binaries
359
360
361
362
363
364
  - task: DownloadBuildArtifacts@0
    displayName: Download R-package
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      artifactName: R-package
      downloadPath: $(Build.SourcesDirectory)/R
Guolin Ke's avatar
Guolin Ke committed
365
  - script: |
366
      python "$(Build.SourcesDirectory)/.nuget/create_nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
367
368
369
370
371
    displayName: 'Create NuGet configuration files'
  - task: NuGetCommand@2
    inputs:
      command: pack
      packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec'
372
      packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
Guolin Ke's avatar
Guolin Ke committed
373
374
  - task: PublishBuildArtifacts@1
    inputs:
375
      pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
376
      artifactName: NuGet
Guolin Ke's avatar
Guolin Ke committed
377
      artifactType: container
378
379
380
381
382
383
384
385
386
387
388
389
  - task: GitHubRelease@0
    displayName: 'Create GitHub Release'
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      gitHubConnection: guolinke
      repositoryName: '$(Build.Repository.Name)'
      action: 'create'
      target: '$(Build.SourceVersion)'
      tagSource: 'auto'
      title: '$(Build.SourceBranchName)'
      assets: |
        $(Build.SourcesDirectory)/binaries/PackageAssets/*
390
        $(Build.SourcesDirectory)/R/R-package/*
391
392
        $(Build.ArtifactStagingDirectory)/nuget/*.nupkg
        $(Build.ArtifactStagingDirectory)/archives/*
393
394
395
      assetUploadMode: 'delete'
      isDraft: true
      isPreRelease: false
Guolin Ke's avatar
Guolin Ke committed
396
      addChangeLog: false