.vsts-ci.yml 12.3 KB
Newer Older
1
2
3
trigger:
  branches:
    include:
4
    - master
5
6
7
  tags:
    include:
    - v*
8
9
pr:
- master
10
variables:
11
  AZURE: 'true'
12
  PYTHON_VERSION: 3.9
13
  CONDA_ENV: test-env
14
15
16
17
resources:
  containers:
  - container: ubuntu1404
    image: lightgbm/vsts-agent:ubuntu-14.04
18
19
20
  - container: ubuntu-latest
    image: 'ubuntu:latest'
    options: "--name ci-container -v /usr/bin/docker:/tmp/docker:ro"
21
22
  - container: rbase
    image: rocker/r-base
23
jobs:
Guolin Ke's avatar
Guolin Ke committed
24
###########################################
25
- job: Linux
Guolin Ke's avatar
Guolin Ke committed
26
###########################################
27
  variables:
28
    COMPILER: gcc
29
    SETUP_CONDA: 'false'
30
31
    OS_NAME: 'linux'
    PRODUCES_ARTIFACTS: 'true'
32
  pool: sh-ubuntu
33
  container: ubuntu1404
34
  strategy:
Guolin Ke's avatar
Guolin Ke committed
35
36
37
38
39
    matrix:
      regular:
        TASK: regular
      sdist:
        TASK: sdist
40
        PYTHON_VERSION: 3.7
Guolin Ke's avatar
Guolin Ke committed
41
42
      bdist:
        TASK: bdist
43
44
45
46
47
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
48
        PYTHON_VERSION: 3.8
49
50
      gpu_source:
        TASK: gpu
Guolin Ke's avatar
Guolin Ke committed
51
        METHOD: source
52
53
      swig:
        TASK: swig
54
55
      cpp_tests:
        TASK: cpp-tests
Guolin Ke's avatar
Guolin Ke committed
56
  steps:
57
  - script: |
58
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
59
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
Nikita Titov's avatar
Nikita Titov committed
60
61
      echo "##vso[task.prependpath]$CONDA/bin"
      AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
62
63
64
65
66
      echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
      LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
      echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
      echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
    displayName: 'Set variables'
67
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
68
    displayName: Setup
69
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
70
    displayName: Test
Nikita Titov's avatar
Nikita Titov committed
71
  - task: PublishBuildArtifacts@1
72
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Nikita Titov's avatar
Nikita Titov committed
73
74
75
76
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
77
###########################################
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
- job: Linux_latest
###########################################
  variables:
    COMPILER: clang
    DEBIAN_FRONTEND: 'noninteractive'
    IN_UBUNTU_LATEST_CONTAINER: 'true'
    OS_NAME: 'linux'
    SETUP_CONDA: 'true'
  pool: sh-ubuntu
  container: ubuntu-latest
  strategy:
    matrix:
      regular:
        TASK: regular
        PYTHON_VERSION: 3.6
      sdist:
        TASK: sdist
      bdist:
        TASK: bdist
97
        PYTHON_VERSION: 3.8
98
99
100
101
102
103
104
105
      inference:
        TASK: if-else
      mpi_source:
        TASK: mpi
        METHOD: source
      mpi_pip:
        TASK: mpi
        METHOD: pip
106
        PYTHON_VERSION: 3.8
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
      mpi_wheel:
        TASK: mpi
        METHOD: wheel
        PYTHON_VERSION: 3.7
      gpu_source:
        TASK: gpu
        METHOD: source
      gpu_pip:
        TASK: gpu
        METHOD: pip
        PYTHON_VERSION: 3.6
      gpu_wheel:
        TASK: gpu
        METHOD: wheel
        PYTHON_VERSION: 3.7
  steps:
  - script: |
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
      CONDA=$HOME/miniconda
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
      AMDAPPSDK_PATH=$BUILD_SOURCESDIRECTORY/AMDAPPSDK
      echo "##vso[task.setvariable variable=AMDAPPSDK_PATH]$AMDAPPSDK_PATH"
      LD_LIBRARY_PATH=$AMDAPPSDK_PATH/lib/x86_64:$LD_LIBRARY_PATH
      echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$LD_LIBRARY_PATH"
      echo "##vso[task.setvariable variable=OPENCL_VENDOR_PATH]$AMDAPPSDK_PATH/etc/OpenCL/vendors"
    displayName: 'Set variables'
  # https://github.com/microsoft/azure-pipelines-agent/issues/2043#issuecomment-687983301
  - script: |
      /tmp/docker exec -t -u 0 ci-container \
      sh -c "apt-get update && apt-get -o Dpkg::Options::="--force-confold" -y install sudo"
    displayName: 'Install sudo'
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
    displayName: Setup
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
    displayName: Test
###########################################
145
146
147
148
149
150
151
152
- job: QEMU_multiarch
###########################################
  variables:
    COMPILER: gcc
    OS_NAME: 'linux'
    PRODUCES_ARTIFACTS: 'true'
  pool:
    vmImage: ubuntu-latest
153
  timeoutInMinutes: 180
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
  strategy:
    matrix:
      bdist:
        TASK: bdist
        ARCH: aarch64
  steps:
  - script: |
      sudo apt-get update
      sudo apt-get install --no-install-recommends -y \
        binfmt-support \
        qemu \
        qemu-user \
        qemu-user-static
    displayName: 'Install QEMU'
  - script: |
      docker run --rm --privileged multiarch/qemu-user-static --reset -p yes
    displayName: 'Enable Docker multi-architecture support'
  - script: |
      export ROOT_DOCKER_FOLDER=/LightGBM
      cat > docker.env <<EOF
      AZURE=$AZURE
      OS_NAME=$OS_NAME
      COMPILER=$COMPILER
      TASK=$TASK
      METHOD=$METHOD
      CONDA_ENV=$CONDA_ENV
      PYTHON_VERSION=$PYTHON_VERSION
      BUILD_DIRECTORY=$ROOT_DOCKER_FOLDER
      LGB_VER=$(head -n 1 VERSION.txt)
      PRODUCES_ARTIFACTS=$PRODUCES_ARTIFACTS
      BUILD_ARTIFACTSTAGINGDIRECTORY=$BUILD_ARTIFACTSTAGINGDIRECTORY
      EOF
      cat > docker-script.sh <<EOF
      export CONDA=\$HOME/miniconda
      export PATH=\$CONDA/bin:\$PATH
      $ROOT_DOCKER_FOLDER/.ci/setup.sh || exit -1
      $ROOT_DOCKER_FOLDER/.ci/test.sh || exit -1
      EOF
      docker run \
        --rm \
        --env-file docker.env \
        -v "$(Build.SourcesDirectory)":"$ROOT_DOCKER_FOLDER" \
        -v "$(Build.ArtifactStagingDirectory)":"$(Build.ArtifactStagingDirectory)" \
        "quay.io/pypa/manylinux2014_$ARCH" \
        /bin/bash $ROOT_DOCKER_FOLDER/docker-script.sh
    displayName: 'Setup and run tests'
  - task: PublishBuildArtifacts@1
    condition: and(succeeded(), in(variables['TASK'], 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
    inputs:
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
      artifactName: PackageAssets
      artifactType: container
###########################################
207
- job: macOS
Guolin Ke's avatar
Guolin Ke committed
208
###########################################
209
  variables:
210
    COMPILER: clang
211
212
    OS_NAME: 'macos'
    PRODUCES_ARTIFACTS: 'true'
213
  pool:
214
    vmImage: 'macOS-10.14'
215
  strategy:
Guolin Ke's avatar
Guolin Ke committed
216
217
218
    matrix:
      regular:
        TASK: regular
219
        PYTHON_VERSION: 3.7
Guolin Ke's avatar
Guolin Ke committed
220
221
      sdist:
        TASK: sdist
222
        PYTHON_VERSION: 3.8
Guolin Ke's avatar
Guolin Ke committed
223
224
      bdist:
        TASK: bdist
225
226
      swig:
        TASK: swig
227
228
      cpp_tests:
        TASK: cpp-tests
Guolin Ke's avatar
Guolin Ke committed
229
  steps:
230
  - script: |
231
      echo "##vso[task.setvariable variable=BUILD_DIRECTORY]$BUILD_SOURCESDIRECTORY"
232
      echo "##vso[task.setvariable variable=LGB_VER]$(head -n 1 VERSION.txt)"
Nikita Titov's avatar
Nikita Titov committed
233
234
235
      CONDA=$AGENT_HOMEDIRECTORY/miniconda
      echo "##vso[task.setvariable variable=CONDA]$CONDA"
      echo "##vso[task.prependpath]$CONDA/bin"
236
      echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64"
237
    displayName: 'Set variables'
238
  - bash: $(Build.SourcesDirectory)/.ci/setup.sh
239
    displayName: Setup
240
  - bash: $(Build.SourcesDirectory)/.ci/test.sh
241
    displayName: Test
Guolin Ke's avatar
Guolin Ke committed
242
  - task: PublishBuildArtifacts@1
243
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
244
    inputs:
245
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
246
247
248
      artifactName: PackageAssets
      artifactType: container
###########################################
249
- job: Windows
Guolin Ke's avatar
Guolin Ke committed
250
###########################################
251
252
253
  pool:
    vmImage: 'vs2017-win2016'
  strategy:
Guolin Ke's avatar
Guolin Ke committed
254
255
256
    matrix:
      regular:
        TASK: regular
257
        PYTHON_VERSION: 3.6
Guolin Ke's avatar
Guolin Ke committed
258
259
      sdist:
        TASK: sdist
260
        PYTHON_VERSION: 3.8
Guolin Ke's avatar
Guolin Ke committed
261
262
      bdist:
        TASK: bdist
263
264
      swig:
        TASK: swig
265
266
      cpp_tests:
        TASK: cpp-tests
Guolin Ke's avatar
Guolin Ke committed
267
  steps:
268
269
270
  - powershell: |
      Write-Host "##vso[task.prependpath]$env:CONDA\Scripts"
    displayName: 'Set Variables'
271
272
  - script: |
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/install_opencl.ps1"
273
    condition: eq(variables['TASK'], 'bdist')
274
    displayName: 'Install OpenCL'
Nikita Titov's avatar
Nikita Titov committed
275
  - script: |
276
277
      cmd /c "conda init powershell"
      cmd /c "powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1"
278
    displayName: Test
Guolin Ke's avatar
Guolin Ke committed
279
  - task: PublishBuildArtifacts@1
280
    condition: and(succeeded(), in(variables['TASK'], 'regular', 'bdist', 'swig'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
Guolin Ke's avatar
Guolin Ke committed
281
    inputs:
282
      pathtoPublish: '$(Build.ArtifactStagingDirectory)'
Guolin Ke's avatar
Guolin Ke committed
283
284
      artifactName: PackageAssets
      artifactType: container
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
###########################################
- job: R_artifact
###########################################
  condition: not(startsWith(variables['Build.SourceBranch'], 'refs/pull/'))
  pool:
    vmImage: 'ubuntu-latest'
  container: rbase
  steps:
  - script: |
      LGB_VER=$(head -n 1 VERSION.txt | sed "s/rc/-/g")
      sh build-cran-package.sh || exit -1
      mv lightgbm_${LGB_VER}.tar.gz $(Build.ArtifactStagingDirectory)/lightgbm-${LGB_VER}-r-cran.tar.gz
    displayName: 'Build CRAN R-package'
  - task: PublishBuildArtifacts@1
    condition: succeeded()
    inputs:
      pathtoPublish: $(Build.ArtifactStagingDirectory)
      artifactName: R-package
      artifactType: container
Guolin Ke's avatar
Guolin Ke committed
304

305
###########################################
306
- job: Package
307
###########################################
Guolin Ke's avatar
Guolin Ke committed
308
309
  dependsOn:
  - Linux
310
  - Linux_latest
311
  - QEMU_multiarch
312
  - macOS
Guolin Ke's avatar
Guolin Ke committed
313
  - Windows
314
  - R_artifact
315
  condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
316
  pool:
317
    vmImage: 'ubuntu-latest'
Guolin Ke's avatar
Guolin Ke committed
318
  steps:
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
  # Create archives with complete source code included (with git submodules)
  - task: ArchiveFiles@2
    displayName: Create zip archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: zip
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_zip.zip'
      replaceExistingArchive: true
  - task: ArchiveFiles@2
    displayName: Create tar.gz archive
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      rootFolderOrFile: $(Build.SourcesDirectory)
      includeRootFolder: false
      archiveType: tar
      tarCompression: gz
      archiveFile: '$(Build.ArtifactStagingDirectory)/archives/LightGBM-complete_source_code_tar_gz.tar.gz'
      replaceExistingArchive: true
Guolin Ke's avatar
Guolin Ke committed
339
340
341
342
343
344
  # Download all agent packages from all previous phases
  - task: DownloadBuildArtifacts@0
    displayName: Download package assets
    inputs:
      artifactName: PackageAssets
      downloadPath: $(Build.SourcesDirectory)/binaries
345
346
347
348
349
350
  - task: DownloadBuildArtifacts@0
    displayName: Download R-package
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      artifactName: R-package
      downloadPath: $(Build.SourcesDirectory)/R
Guolin Ke's avatar
Guolin Ke committed
351
  - script: |
352
      python "$(Build.SourcesDirectory)/.nuget/create_nuget.py" "$(Build.SourcesDirectory)/binaries/PackageAssets"
353
354
355
356
357
    displayName: 'Create NuGet configuration files'
  - task: NuGetCommand@2
    inputs:
      command: pack
      packagesToPack: '$(Build.SourcesDirectory)/.nuget/*.nuspec'
358
      packDestination: '$(Build.ArtifactStagingDirectory)/nuget'
Guolin Ke's avatar
Guolin Ke committed
359
360
  - task: PublishBuildArtifacts@1
    inputs:
361
      pathtoPublish: '$(Build.ArtifactStagingDirectory)/nuget'
362
      artifactName: NuGet
Guolin Ke's avatar
Guolin Ke committed
363
      artifactType: container
364
365
366
367
368
369
370
371
372
373
374
375
  - task: GitHubRelease@0
    displayName: 'Create GitHub Release'
    condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))
    inputs:
      gitHubConnection: guolinke
      repositoryName: '$(Build.Repository.Name)'
      action: 'create'
      target: '$(Build.SourceVersion)'
      tagSource: 'auto'
      title: '$(Build.SourceBranchName)'
      assets: |
        $(Build.SourcesDirectory)/binaries/PackageAssets/*
376
        $(Build.SourcesDirectory)/R/R-package/*
377
378
        $(Build.ArtifactStagingDirectory)/nuget/*.nupkg
        $(Build.ArtifactStagingDirectory)/archives/*
379
380
381
      assetUploadMode: 'delete'
      isDraft: true
      isPreRelease: false
Guolin Ke's avatar
Guolin Ke committed
382
      addChangeLog: false