.vsts-ci.yml 12.7 KB
Newer Older
1
2
3
trigger:
  branches:
    include:
4
    - master
5
6
7
  tags:
    include:
    - v*
8
9
pr:
- master
10
variables:
11
  AZURE: 'true'
12
  PYTHON_VERSION: 3.9
13
  CONDA_ENV: test-env
14
15
16
17
resources:
  containers:
  - container: ubuntu1404
    image: lightgbm/vsts-agent:ubuntu-14.04
18
19
20
  - container: ubuntu-latest
    image: 'ubuntu:latest'
    options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
21
22
  - container: rbase
    image: rocker/r-base
23
jobs:
Guolin Ke's avatar
Guolin Ke committed
24
###########################################
25
- job: Linux
Guolin Ke's avatar
Guolin Ke committed
26
###########################################
27
  variables:
28
    COMPILER: gcc
29
    SETUP_CONDA: 'false'
30
31
    OS_NAME: 'linux'
    PRODUCES_ARTIFACTS: 'true'
32
  pool: sh-ubuntu
33
  container: ubuntu1404
34
  strategy:
Guolin Ke's avatar
Guolin Ke committed
35
36
37
38
39
    matrix:
      regular:
        TASK: regular
      sdist:
        TASK: sdist
40
        PYTHON_VERSION: 3.7
Guolin Ke's avatar
Guolin Ke committed
41
42
      bdist:
        TASK: bdist
43
44
45
46
47
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
48
        PYTHON_VERSION: 3.8
49
50
      gpu_source:
        TASK: gpu
Guolin Ke's avatar
Guolin Ke committed
51
        METHOD: source
52
53
      swig:
        TASK: swig
Guolin Ke's avatar
Guolin Ke committed
54
  steps:
55
  - script: |
56
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
57
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
Nikita Titov's avatar
Nikita Titov committed
58
59
      echo "##vso[task.prependpath]$CONDA/bin"
      AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
60
61
62
63
64
      echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
      LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
      echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
      echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
    displayName: 'Set variables'
65
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
66
    displayName: Setup
67
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
68
    displayName: Test
Nikita Titov's avatar
Nikita Titov committed
69
  - task: PublishBuildArtifacts@1
70
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Nikita Titov's avatar
Nikita Titov committed
71
72
73
74
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
75
###########################################
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
- job: Linux_latest
###########################################
  variables:
    COMPILER: clang
    DEBIAN_FRONTEND: 'noninteractive'
    IN_UBUNTU_LATEST_CONTAINER: 'true'
    OS_NAME: 'linux'
    SETUP_CONDA: 'true'
  pool: sh-ubuntu
  container: ubuntu-latest
  strategy:
    matrix:
      regular:
        TASK: regular
        PYTHON_VERSION: 3.6
      sdist:
        TASK: sdist
      bdist:
        TASK: bdist
95
        PYTHON_VERSION: 3.8
96
97
98
99
100
101
102
103
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
      mpi_pip:
        TASK: mpi
        METHOD: pip
104
        PYTHON_VERSION: 3.8
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
      mpi_wheel:
        TASK: mpi
        METHOD: wheel
        PYTHON_VERSION: 3.7
      gpu_source:
        TASK: gpu
        METHOD: source
      gpu_pip:
        TASK: gpu
        METHOD: pip
        PYTHON_VERSION: 3.6
      gpu_wheel:
        TASK: gpu
        METHOD: wheel
        PYTHON_VERSION: 3.7
120
121
122
      cpp_tests:
        TASK: cpp-tests
        METHOD: with-sanitizers
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
  steps:
  - script: |
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
      CONDA=$HOME/miniconda
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
      AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
      echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
      LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
      echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
      echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
    displayName: 'Set variables'
  # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
  - script: |
      /tmp/docker exec -t -u 0 ci-container \
      sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
    displayName: 'Install sudo'
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
    displayName: Setup
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
    displayName: Test
###########################################
146
147
148
149
150
151
152
153
- job: QEMU_multiarch
###########################################
  variables:
    COMPILER: gcc
    OS_NAME: 'linux'
    PRODUCES_ARTIFACTS: 'true'
  pool:
    vmImage: ubuntu-latest
154
  timeoutInMinutes: 180
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
  strategy:
    matrix:
      bdist:
        TASK: bdist
        ARCH: aarch64
  steps:
  - script: |
      sudo apt-get update
      sudo apt-get install --no-install-recommends -y \
        binfmt-support \
        qemu \
        qemu-user \
        qemu-user-static
    displayName: 'Install QEMU'
  - script: |
      docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
    displayName: 'Enable Docker multi-architecture support'
  - script: |
      export ROOT_DOCKER_FOLDER=/LightGBM
      cat > docker.env <<EOF
      AZURE=$AZURE
      OS_NAME=$OS_NAME
      COMPILER=$COMPILER
      TASK=$TASK
      METHOD=$METHOD
      CONDA_ENV=$CONDA_ENV
      PYTHON_VERSION=$PYTHON_VERSION
      BUILD_DIRECTORY=$ROOT_DOCKER_FOLDER
      LGB_VER=$(head -n 1 VERSION.txt)
      PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS
      BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY
      EOF
      cat > docker-script.sh <<EOF
      export CONDA=\$HOME/miniconda
      export PATH=\$CONDA/bin:\$PATH
      $ROOT_DOCKER_FOLDER/.ci/setup.sh || exit -1
      $ROOT_DOCKER_FOLDER/.ci/test.sh || exit -1
      EOF
193
194
      IMAGE_URI="quay.io/pypa/manylinux2014_${ARCH}"
      docker pull "${IMAGE_URI}" || exit -1
195
      PLATFORM=$(docker inspect --format='{{.Os}}/{{.Architecture}}' "${IMAGE_URI}") || exit -1
196
      echo "detected image platform: ${PLATFORM}"
197
      docker run \
198
        --platform "${PLATFORM}" \
199
200
201
202
        --rm \
        --env-file docker.env \
        -v "$(Build.SourcesDirectory)":"$ROOT_DOCKER_FOLDER" \
        -v "$(Build.ArtifactStagingDirectory)":"$(Build.ArtifactStagingDirectory)" \
203
        "${IMAGE_URI}" \
204
205
206
207
208
209
210
211
212
        /bin/bash $ROOT_DOCKER_FOLDER/docker-script.sh
    displayName: 'Setup and run tests'
  - task: PublishBuildArtifacts@1
    condition: and(succeeded(), in(variables['TASK'], 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
###########################################
213
- job: macOS
Guolin Ke's avatar
Guolin Ke committed
214
###########################################
215
  variables:
216
    COMPILER: clang
217
218
    OS_NAME: 'macos'
    PRODUCES_ARTIFACTS: 'true'
219
  pool:
220
    vmImage: 'macOS-10.14'
221
  strategy:
Guolin Ke's avatar
Guolin Ke committed
222
223
224
    matrix:
      regular:
        TASK: regular
225
        PYTHON_VERSION: 3.7
Guolin Ke's avatar
Guolin Ke committed
226
227
      sdist:
        TASK: sdist
228
        PYTHON_VERSION: 3.8
Guolin Ke's avatar
Guolin Ke committed
229
230
      bdist:
        TASK: bdist
231
232
      swig:
        TASK: swig
233
234
      cpp_tests:
        TASK: cpp-tests
235
236
        METHOD: with-sanitizers
        SANITIZERS: "address;undefined"
Guolin Ke's avatar
Guolin Ke committed
237
  steps:
238
  - script: |
239
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
240
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
Nikita Titov's avatar
Nikita Titov committed
241
242
243
      CONDA=$AGENT_HOMEDIRECTORY/miniconda
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
244
      echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64"
245
    displayName: 'Set variables'
246
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
247
    displayName: Setup
248
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
249
    displayName: Test
Guolin Ke's avatar
Guolin Ke committed
250
  - task: PublishBuildArtifacts@1
251
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
252
    inputs:
253
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
254
255
256
      artifactName: PackageAssets
      artifactType: container
###########################################
257
- job: Windows
Guolin Ke's avatar
Guolin Ke committed
258
###########################################
259
260
261
  pool:
    vmImage: 'vs2017-win2016'
  strategy:
Guolin Ke's avatar
Guolin Ke committed
262
263
264
    matrix:
      regular:
        TASK: regular
265
        PYTHON_VERSION: 3.6
Guolin Ke's avatar
Guolin Ke committed
266
267
      sdist:
        TASK: sdist
268
        PYTHON_VERSION: 3.8
Guolin Ke's avatar
Guolin Ke committed
269
270
      bdist:
        TASK: bdist
271
272
      swig:
        TASK: swig
273
274
      cpp_tests:
        TASK: cpp-tests
Guolin Ke's avatar
Guolin Ke committed
275
  steps:
276
277
278
  - powershell: |
      Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
    displayName: 'Set Variables'
279
280
  - script: |
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install_opencl.ps1"
281
    condition: eq(variables['TASK'], 'bdist')
282
    displayName: 'Install OpenCL'
Nikita Titov's avatar
Nikita Titov committed
283
  - script: |
284
285
      cmd /c "conda init powershell"
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1"
286
    displayName: Test
Guolin Ke's avatar
Guolin Ke committed
287
  - task: PublishBuildArtifacts@1
288
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
289
    inputs:
290
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
291
292
      artifactName: PackageAssets
      artifactType: container
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
###########################################
- job: R_artifact
###########################################
  condition: not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
  pool:
    vmImage: 'ubuntu-latest'
  container: rbase
  steps:
  - script: |
      LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g")
      sh build-cran-package.sh || exit -1
      mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz
    displayName: 'Build CRAN R-package'
  - task: PublishBuildArtifacts@1
    condition: succeeded()
    inputs:
      pathtoPublish: $(Build.ArtifactStagingDirectory)
      artifactName: R-package
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
312

313
###########################################
314
- job: Package
315
###########################################
Guolin Ke's avatar
Guolin Ke committed
316
317
  dependsOn:
  - Linux
318
  - Linux_latest
319
  - QEMU_multiarch
320
  - macOS
Guolin Ke's avatar
Guolin Ke committed
321
  - Windows
322
  - R_artifact
323
  condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
324
  pool:
325
    vmImage: 'ubuntu-latest'
Guolin Ke's avatar
Guolin Ke committed
326
  steps:
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
  # Create archives with complete source code included (with git submodules)
  - task: ArchiveFiles@2
    displayName: Create zip archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: zip
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_zip.zip'
      replaceExistingArchive: true
  - task: ArchiveFiles@2
    displayName: Create tar.gz archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: tar
      tarCompression: gz
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_tar_gz.tar.gz'
      replaceExistingArchive: true
Guolin Ke's avatar
Guolin Ke committed
347
348
349
350
351
352
  # Download all agent packages from all previous phases
  - task: DownloadBuildArtifacts@0
    displayName: Download package assets
    inputs:
      artifactName: PackageAssets
      downloadPath: $(Build.SourcesDirectory)/binaries
353
354
355
356
357
358
  - task: DownloadBuildArtifacts@0
    displayName: Download R-package
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      artifactName: R-package
      downloadPath: $(Build.SourcesDirectory)/R
Guolin Ke's avatar
Guolin Ke committed
359
  - script: |
360
      python "$(Build.SourcesDirectory)/.nuget/create_nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
361
362
363
364
365
    displayName: 'Create NuGet configuration files'
  - task: NuGetCommand@2
    inputs:
      command: pack
      packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec'
366
      packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
Guolin Ke's avatar
Guolin Ke committed
367
368
  - task: PublishBuildArtifacts@1
    inputs:
369
      pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
370
      artifactName: NuGet
Guolin Ke's avatar
Guolin Ke committed
371
      artifactType: container
372
373
374
375
376
377
378
379
380
381
382
383
  - task: GitHubRelease@0
    displayName: 'Create GitHub Release'
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      gitHubConnection: guolinke
      repositoryName: '$(Build.Repository.Name)'
      action: 'create'
      target: '$(Build.SourceVersion)'
      tagSource: 'auto'
      title: '$(Build.SourceBranchName)'
      assets: |
        $(Build.SourcesDirectory)/binaries/PackageAssets/*
384
        $(Build.SourcesDirectory)/R/R-package/*
385
386
        $(Build.ArtifactStagingDirectory)/nuget/*.nupkg
        $(Build.ArtifactStagingDirectory)/archives/*
387
388
389
      assetUploadMode: 'delete'
      isDraft: true
      isPreRelease: false
Guolin Ke's avatar
Guolin Ke committed
390
      addChangeLog: false