[ci] added swig compilation on macOS and Windows with artifacts (#2170)

* compile SWIG artifact on Windows

* compile SWIG artifact on Windows

* combined swig and sdist jobs for Linux

* added conditions for artifacts publishing

* added SWIG artifact compilation on macOS

* hotfix

* test: switch compilers

* harsh workaround for OpenMP library conflict

* switch compilers back

* consistent order of tasks in condition

* dummy commit to trigger CI and fix CLA

.ci/setup.sh

@@ -22,6 +22,9 @@ if [[ $OS_NAME == "macos" ]]; then
     if [[ $TASK == "mpi" ]]; then
         brew install open-mpi
     fi
+    if [[ $AZURE == "true" ]] && [[ $TASK == "sdist" ]]; then
+        brew install https://raw.githubusercontent.com/Homebrew/homebrew-core/f3544543a3115023fc7ca962c21d14b443f419d0/Formula/swig.rb  # swig 3.0.12
+    fi
     wget -q -O conda.sh https://repo.continuum.io/miniconda/Miniconda${PYTHON_VERSION:0:1}-latest-MacOSX-x86_64.sh
 else  # Linux
     if [[ $AZURE == "true" ]] && [[ $COMPILER == "clang" ]]; then

.ci/test.sh

@@ -18,19 +18,6 @@ else
     CMAKE_OPTS=()
 fi
 
-if [[ $AZURE == "true" ]] && [[ $OS_NAME == "linux" ]] && [[ $TASK == "swig" ]]; then
-    mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build
-    cmake -DUSE_SWIG=ON "${CMAKE_OPTS[@]}" ..
-    make -j4 || exit -1
-    if [[ $COMPILER == "gcc" ]]; then
-        objdump -T $BUILD_DIRECTORY/lib_lightgbm.so > $BUILD_DIRECTORY/objdump.log || exit -1
-        objdump -T $BUILD_DIRECTORY/lib_lightgbm_swig.so >> $BUILD_DIRECTORY/objdump.log || exit -1
-        python $BUILD_DIRECTORY/helpers/check_dynamic_dependencies.py $BUILD_DIRECTORY/objdump.log || exit -1
-    fi
-    cp $BUILD_DIRECTORY/build/lightgbmlib.jar $BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib.jar
-    exit 0
-fi
-
 conda create -q -y -n $CONDA_ENV python=$PYTHON_VERSION
 source activate $CONDA_ENV
 
@@ -91,6 +78,19 @@ if [[ $TASK == "sdist" ]]; then
     pip install --user $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz -v || exit -1
     if [[ $AZURE == "true" ]]; then
         cp $BUILD_DIRECTORY/python-package/dist/lightgbm-$LGB_VER.tar.gz $BUILD_ARTIFACTSTAGINGDIRECTORY
+        mkdir $BUILD_DIRECTORY/build && cd $BUILD_DIRECTORY/build
+        if [[ $OS_NAME == "macos" ]]; then
+            cmake -DUSE_SWIG=ON -DAPPLE_OUTPUT_DYLIB=ON "${CMAKE_OPTS[@]}" ..
+        else
+            cmake -DUSE_SWIG=ON "${CMAKE_OPTS[@]}" ..
+        fi
+        make -j4 || exit -1
+        if [[ $OS_NAME == "linux" ]] && [[ $COMPILER == "gcc" ]]; then
+            objdump -T $BUILD_DIRECTORY/lib_lightgbm.so > $BUILD_DIRECTORY/objdump.log || exit -1
+            objdump -T $BUILD_DIRECTORY/lib_lightgbm_swig.so >> $BUILD_DIRECTORY/objdump.log || exit -1
+            python $BUILD_DIRECTORY/helpers/check_dynamic_dependencies.py $BUILD_DIRECTORY/objdump.log || exit -1
+        fi
+        cp $BUILD_DIRECTORY/build/lightgbmlib.jar $BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_$OS_NAME.jar
     fi
     pytest $BUILD_DIRECTORY/tests/python_package_test || exit -1
     exit 0

.ci/test_windows.ps1

@@ -20,6 +20,15 @@ elseif ($env:TASK -eq "sdist") {
   cd $env:BUILD_SOURCESDIRECTORY/python-package
   python setup.py sdist --formats gztar ; Check-Output $?
   cd dist; pip install @(Get-ChildItem *.gz) -v ; Check-Output $?
+
+  $env:JAVA_HOME = $env:JAVA_HOME_8_X64  # there is pre-installed Zulu OpenJDK-8 somewhere
+  Invoke-WebRequest -Uri "https://sourceforge.net/projects/swig/files/swigwin/swigwin-3.0.12/swigwin-3.0.12.zip/download" -OutFile $env:BUILD_SOURCESDIRECTORY/swig/swigwin.zip -UserAgent "NativeHost"
+  Add-Type -AssemblyName System.IO.Compression.FileSystem
+  [System.IO.Compression.ZipFile]::ExtractToDirectory("$env:BUILD_SOURCESDIRECTORY/swig/swigwin.zip", "$env:BUILD_SOURCESDIRECTORY/swig")
+  $env:PATH += ";$env:BUILD_SOURCESDIRECTORY/swig/swigwin-3.0.12"
+  mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build
+  cmake -A x64 -DUSE_SWIG=ON .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $?
+  cp $env:BUILD_SOURCESDIRECTORY/build/lightgbmlib.jar $env:BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_win.jar
 }
 elseif ($env:TASK -eq "bdist") {
   cd $env:BUILD_SOURCESDIRECTORY/python-package

.vsts-ci.yml

@@ -15,7 +15,7 @@ jobs:
     vmImage: 'ubuntu-16.04'
   container: ubuntu1404
   strategy:
-    maxParallel: 7
+    maxParallel: 6
     matrix:
       regular:
         TASK: regular
@@ -25,8 +25,6 @@ jobs:
       bdist:
         TASK: bdist
         PYTHON_VERSION: 3.6
-      swig:
-        TASK: swig
       inference:
         TASK: if-else
       mpi_source:
@@ -56,7 +54,7 @@ jobs:
   - bash: $(Build.SourcesDirectory)/.ci/test.sh
     displayName: Test
   - task: PublishBuildArtifacts@1
-    condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
+    condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
     inputs:
       pathtoPublish: '$(Build.ArtifactStagingDirectory)'
       artifactName: PackageAssets
@@ -89,13 +87,14 @@ jobs:
       CONDA=$AGENT_HOMEDIRECTORY/miniconda
       echo "##vso[task.setvariable variable=CONDA]$CONDA"
       echo "##vso[task.prependpath]$CONDA/bin"
+      echo "##vso[task.setvariable variable=JAVA_HOME]$JAVA_HOME_8_X64"
     displayName: 'Set variables'
   - bash: $(Build.SourcesDirectory)/.ci/setup.sh
     displayName: Setup
   - bash: $(Build.SourcesDirectory)/.ci/test.sh
     displayName: Test
   - task: PublishBuildArtifacts@1
-    condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
+    condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
     inputs:
       pathtoPublish: '$(Build.ArtifactStagingDirectory)' 
       artifactName: PackageAssets
@@ -126,7 +125,7 @@ jobs:
       cmd /c "activate %CONDA_ENV% & powershell -ExecutionPolicy Bypass -File %BUILD_SOURCESDIRECTORY%/.ci/test_windows.ps1"
     displayName: Test
   - task: PublishBuildArtifacts@1
-    condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
+    condition: and(succeeded(), in(variables['TASK'], 'regular', 'sdist', 'bdist'), not(startsWith(variables['Build.SourceBranch'], 'refs/pull/')))
     inputs:
       pathtoPublish: '$(Build.ArtifactStagingDirectory)'
       artifactName: PackageAssets