[ci] rebalanced Windows tests (#1660)

* refined Appveyor

* run all tests at Appveyor

* refined VSTS Windows part

* return back old MinGW version

* fixed weird MinGW bug

* don't test Python 3.7 at Windows

* disable tree rendering at Appveyor

* give names to scripts in VSTS Windows and artifacts parts

.appveyor.yml

 version: 2.1.2.{build}
 
-configuration:  # a trick to construct a build matrix
-  - 3.5
+image: Visual Studio 2015
+platform: x64
+configuration:  # a trick to construct a build matrix with multiple Python versions
   - 3.6
 
 environment:
   matrix:
-    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
-      COMPILER: MSVC
-    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
-      COMPILER: MINGW
+    - COMPILER: MSVC
+    - COMPILER: MINGW
 
 clone_depth: 50
 
 install:
   - git submodule update --init --recursive  # get `compute` folder
   - set PATH=%PATH:C:\Program Files\Git\usr\bin;=%  # delete sh.exe from PATH (mingw32-make fix)
-  - set PATH=C:\mingw-w64\x86_64-6.3.0-posix-seh-rt_v5-rev1\mingw64\bin;%PATH%
+  - set PATH=C:\mingw-w64\x86_64-7.3.0-posix-seh-rt_v5-rev0\mingw64\bin;%PATH%
   - set PYTHON_VERSION=%CONFIGURATION%
   - ps: >-
       switch ($env:PYTHON_VERSION) {
@@ -34,33 +33,25 @@ install:
   - activate test-env
 
 build_script:
-  - mkdir %APPVEYOR_BUILD_FOLDER%\build && cd %APPVEYOR_BUILD_FOLDER%\build
-  - cmake -DCMAKE_GENERATOR_PLATFORM=x64 .. && cmake --build . --target ALL_BUILD --config Release
-
-test_script:
-  - pytest %APPVEYOR_BUILD_FOLDER%\tests\c_api_test\test_.py
-  - cd %APPVEYOR_BUILD_FOLDER%\python-package && python setup.py sdist --formats gztar
+  - cd %APPVEYOR_BUILD_FOLDER%\python-package
   - IF "%COMPILER%"=="MINGW" (
-    pip install %APPVEYOR_BUILD_FOLDER%\python-package\dist\lightgbm-%LGB_VER%.tar.gz --install-option=--mingw -v)
+    python setup.py install --mingw)
     ELSE (
-    pip install %APPVEYOR_BUILD_FOLDER%\python-package\dist\lightgbm-%LGB_VER%.tar.gz -v)
+    python setup.py install)
+
+test_script:
   - pytest %APPVEYOR_BUILD_FOLDER%\tests\python_package_test
   - cd %APPVEYOR_BUILD_FOLDER%\examples\python-guide
   - ps: >-
       @("import matplotlib", "matplotlib.use('Agg')") + (Get-Content "plot_example.py") | Set-Content "plot_example.py"  # prevent interactive window mode
+      (Get-Content "plot_example.py").replace('graph.render(view=True)', 'graph.render(view=False)') | Set-Content "plot_example.py"
   - ps: >-
       foreach ($file in @(Get-ChildItem *.py)) {
         python $file
         if ($LastExitCode -ne 0) { $host.SetShouldExit($LastExitCode) }
       }  # run all examples
   - IF "%COMPILER%"=="MINGW" appveyor exit  # skip all further steps
-  - cd %APPVEYOR_BUILD_FOLDER%\python-package && python setup.py bdist_wheel --plat-name=win-amd64 --universal
 
 artifacts:
-- path: Release/lib_lightgbm.dll
+- path: python-package\compile\windows\x64\DLL\lib_lightgbm.dll
   name: Library
-- path: Release/lightgbm.exe
-  name: Exe
-- path: python-package/dist/*
-  name: Pip
-

.vsts-ci.yml

@@ -100,60 +100,26 @@ phases:
 ###########################################
   queue:
     name: 'Hosted VS2017'
-    parallel: 4
+    parallel: 3
     matrix:
       regular:
         TASK: regular
+        PYTHON_VERSION: 3.5
       sdist:
         TASK: sdist
         PYTHON_VERSION: 2.7
       bdist:
         TASK: bdist
-        PYTHON_VERSION: 3.5
-      # mingw:
-      #   TASK: mingw
+        PYTHON_VERSION: 3.6
   steps:
   - task: CondaEnvironment@0
     inputs:
+      updateConda: true
       environmentName: $(CONDA_ENV)
-      packageSpecs: 'python=$(PYTHON_VERSION)'
+      packageSpecs: 'python=$(PYTHON_VERSION) numpy nose scipy scikit-learn pandas matplotlib python-graphviz pytest'
       createOptions: '-q'
-  - powershell: |
-      conda install -q -y -n $env:CONDA_ENV numpy nose scipy scikit-learn pandas matplotlib python-graphviz pytest
-      if ("$env:TASK" -eq "regular") {
-        mkdir build; cd build
-        cmake -DCMAKE_GENERATOR_PLATFORM=x64 .. ; cmake --build . --target ALL_BUILD --config Release
-        cd ../python-package; python setup.py install -p
-        cd ..
-        pytest tests/c_api_test/test_.py
-        cp Release/lib_lightgbm.dll $env:BUILD_ARTIFACTSTAGINGDIRECTORY
-        cp Release/lightgbm.exe $env:BUILD_ARTIFACTSTAGINGDIRECTORY
-      } 
-      elseif ("$env:TASK" -eq "sdist"){
-        cd python-package; python setup.py sdist --formats gztar
-        cd dist; pip install @(Get-ChildItem *.gz) -v 
-        cd ../..
-      }
-      # elseif ("$env:TASK" -eq "mingw"){
-      #   cd python-package; python setup.py sdist --formats gztar
-      #   cd dist; pip install @(Get-ChildItem *.gz) --install-option=--mingw -v
-      #   cd ../..
-      # }
-      else {
-        cd python-package
-        python setup.py bdist_wheel --plat-name=win-amd64 --universal
-        cd dist; pip install @(Get-ChildItem *.whl)
-        cp @(Get-ChildItem *.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
-        cd ../..
-      }
-      pytest tests/python_package_test
-      cd examples/python-guide
-      @("import matplotlib", "matplotlib.use('Agg')") + (Get-Content "plot_example.py") | Set-Content "plot_example.py"
-      (Get-Content "plot_example.py").replace('graph.render(view=True)', 'graph.render(view=False)') | Set-Content "plot_example.py"
-      foreach ($file in @(Get-ChildItem *.py)) {
-        python $file
-        if ($LastExitCode -ne 0) { $host.SetShouldExit($LastExitCode) }
-      }  # run all examples
+  - powershell: $(Build.SourcesDirectory)/.vsts-ci/test_windows.ps1
+    displayName: Test
   - task: PublishBuildArtifacts@1
     condition: and(succeeded(), startsWith(variables['Build.SourceBranch'], 'refs/tags/'))
     inputs:
@@ -180,12 +146,14 @@ phases:
       downloadPath: $(Build.SourcesDirectory)/binaries
   - powershell: |
       $client = new-object System.Net.WebClient
-      $client.DownloadFile("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe",".nuget/nuget.exe")
+      $client.DownloadFile("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", "$(Build.SourcesDirectory)/.nuget/nuget.exe")
+    displayName: 'Download NuGet application'
   - script: |
-      cd .nuget
+      cd %BUILD_SOURCESDIRECTORY%/.nuget
       python create_nuget.py %BUILD_SOURCESDIRECTORY%/binaries/PackageAssets
       nuget.exe pack LightGBM.nuspec
       xcopy *.nupkg %BUILD_ARTIFACTSTAGINGDIRECTORY%
+    displayName: 'Build NuGet package'
   - task: PublishBuildArtifacts@1
     inputs:
       pathtoPublish: '$(Build.ArtifactStagingDirectory)'

.vsts-ci/test_windows.ps1

+function Check-Output {
+ param( [int]$ExitCode )
+ if ($ExitCode -ne 0) {
+    $host.SetShouldExit($ExitCode)
+    Exit -1
+  }
+}
+
+if ($env:TASK -eq "regular") {
+  mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build
+  cmake -DCMAKE_GENERATOR_PLATFORM=x64 .. ; cmake --build . --target ALL_BUILD --config Release ; Check-Output $LastExitCode
+  cd $env:BUILD_SOURCESDIRECTORY/python-package
+  python setup.py install --precompile ; Check-Output $LastExitCode
+  cp $env:BUILD_SOURCESDIRECTORY/Release/lib_lightgbm.dll $env:BUILD_ARTIFACTSTAGINGDIRECTORY
+  cp $env:BUILD_SOURCESDIRECTORY/Release/lightgbm.exe $env:BUILD_ARTIFACTSTAGINGDIRECTORY
+}
+elseif ($env:TASK -eq "sdist") {
+  cd $env:BUILD_SOURCESDIRECTORY/python-package
+  python setup.py sdist --formats gztar ; Check-Output $LastExitCode
+  cd dist; pip install @(Get-ChildItem *.gz) -v ; Check-Output $LastExitCode
+}
+elseif ($env:TASK -eq "bdist") {
+  cd $env:BUILD_SOURCESDIRECTORY/python-package
+  python setup.py bdist_wheel --plat-name=win-amd64 --universal ; Check-Output $LastExitCode
+  cd dist; pip install @(Get-ChildItem *.whl) ; Check-Output $LastExitCode
+  cp @(Get-ChildItem *.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
+}
+
+$tests = $env:BUILD_SOURCESDIRECTORY + $(If ($env:TASK -eq "sdist") {"/tests/python_package_test"} Else {"/tests"})  # cannot test C API with "sdist" task
+pytest $tests ; Check-Output $LastExitCode
+
+if ($env:TASK -eq "regular") {
+  cd $env:BUILD_SOURCESDIRECTORY/examples/python-guide
+  @("import matplotlib", "matplotlib.use('Agg')") + (Get-Content "plot_example.py") | Set-Content "plot_example.py"
+  (Get-Content "plot_example.py").replace('graph.render(view=True)', 'graph.render(view=False)') | Set-Content "plot_example.py"
+  foreach ($file in @(Get-ChildItem *.py)) {
+    python $file ; Check-Output $LastExitCode
+  }  # run all examples
+}