Unverified Commit dc0ed538 authored by Nikita Titov's avatar Nikita Titov Committed by GitHub
Browse files

[ci] check PowerShell scripts with PSScriptAnalyzer (part 1) (#6704)

* introdure PSScriptAnalyzer

* revert workflow

* run PSScriptAnalyzer before conda installation
parent 4a60a53c
$settings = @{
Severity = @(
'Information',
'Warning',
'Error'
)
IncludeDefaultRules = $true
# Additional rules that are disabled by default
Rules = @{
PSAvoidExclaimOperator = @{
Enable = $true
}
PSAvoidLongLines = @{
Enable = $true
MaximumLineLength = 120
}
PSAvoidSemicolonsAsLineTerminators = @{
Enable = $true
}
PSPlaceCloseBrace = @{
Enable = $true
NoEmptyLineBefore = $true
IgnoreOneLineBlock = $true
NewLineAfter = $false
}
PSPlaceOpenBrace = @{
Enable = $true
OnSameLine = $true
NewLineAfter = $true
IgnoreOneLineBlock = $true
}
PSUseConsistentIndentation = @{
Enable = $true
IndentationSize = 4
PipelineIndentation = 'IncreaseIndentationAfterEveryPipeline'
Kind = 'space'
}
PSUseConsistentWhitespace = @{
Enable = $true
CheckInnerBrace = $true
CheckOpenBrace = $true
CheckOpenParen = $true
CheckOperator = $true
CheckSeparator = $true
CheckPipe = $true
CheckPipeForRedundantWhitespace = $true
CheckParameter = $true
IgnoreAssignmentOperatorInsideHashTable = $false
}
PSUseCorrectCasing = @{
Enable = $true
}
}
}
Invoke-ScriptAnalyzer -Path "$env:BUILD_DIRECTORY/.ci" -Recurse -EnableExit -Settings $settings
loadNamespace("lintr")
args <- commandArgs(
......
# Download a file and retry upon failure. This looks like
# an infinite loop but CI-level timeouts will kill it
function Download-File-With-Retries {
function Get-File-With-Tenacity {
param(
[string]$url,
[string]$destfile
[Parameter(Mandatory = $true)][string]$url,
[Parameter(Mandatory = $true)][string]$destfile
)
$ProgressPreference = "SilentlyContinue" # progress bar bug extremely slows down download speed
do {
Write-Output "Downloading ${url}"
sleep 5;
sleep 5
Invoke-WebRequest -Uri $url -OutFile $destfile
} while(!$?);
} while (-not $?)
}
# External utilities like R.exe / Rscript.exe writing to stderr (even for harmless
......@@ -20,9 +20,9 @@ function Download-File-With-Retries {
# Using standard PowerShell redirection does not work to avoid these errors.
# This function uses R's built-in redirection mechanism, sink(). Any place where
# this function is used is a command that writes harmless messages to stderr
function Run-R-Code-Redirect-Stderr {
function Invoke-R-Code-Redirect-Stderr {
param(
[string]$rcode
[Parameter(Mandatory = $true)][string]$rcode
)
$decorated_code = "out_file <- file(tempfile(), open = 'wt'); sink(out_file, type = 'message'); $rcode; sink()"
Rscript --vanilla -e $decorated_code
......@@ -30,10 +30,13 @@ function Run-R-Code-Redirect-Stderr {
# Remove all items matching some pattern from PATH environment variable
function Remove-From-Path {
[CmdletBinding(SupportsShouldProcess)]
param(
[string]$pattern_to_remove
[Parameter(Mandatory = $true)][string]$pattern_to_remove
)
if ($PSCmdlet.ShouldProcess($env:PATH, "Removing ${pattern_to_remove}")) {
$env:PATH = ($env:PATH.Split(';') | Where-Object { $_ -notmatch "$pattern_to_remove" }) -join ';'
}
}
# remove some details that exist in the GitHub Actions images which might
......@@ -87,7 +90,7 @@ if ($env:R_MAJOR_VERSION -eq "3") {
$env:R_WINDOWS_VERSION = "4.3.1"
} else {
Write-Output "[ERROR] Unrecognized R version: $env:R_VERSION"
Check-Output $false
Assert-Output $false
}
$env:CMAKE_VERSION = "3.30.0"
......@@ -120,29 +123,29 @@ tzutil /s "GMT Standard Time"
# download R, RTools and CMake
Write-Output "Downloading R, Rtools and CMake"
Download-File-With-Retries -url "$env:CRAN_MIRROR/bin/windows/base/old/$env:R_WINDOWS_VERSION/R-$env:R_WINDOWS_VERSION-win.exe" -destfile "R-win.exe"
Download-File-With-Retries -url "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/$env:RTOOLS_EXE_FILE" -destfile "Rtools.exe"
Download-File-With-Retries -url "https://github.com/Kitware/CMake/releases/download/v$env:CMAKE_VERSION/cmake-$env:CMAKE_VERSION-windows-x86_64.zip" -destfile "$env:CMAKE_PATH/cmake.zip"
Get-File-With-Tenacity -url "$env:CRAN_MIRROR/bin/windows/base/old/$env:R_WINDOWS_VERSION/R-$env:R_WINDOWS_VERSION-win.exe" -destfile "R-win.exe"
Get-File-With-Tenacity -url "https://github.com/microsoft/LightGBM/releases/download/v2.0.12/$env:RTOOLS_EXE_FILE" -destfile "Rtools.exe"
Get-File-With-Tenacity -url "https://github.com/Kitware/CMake/releases/download/v$env:CMAKE_VERSION/cmake-$env:CMAKE_VERSION-windows-x86_64.zip" -destfile "$env:CMAKE_PATH/cmake.zip"
# Install R
Write-Output "Installing R"
Start-Process -FilePath R-win.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /DIR=$env:R_LIB_PATH/R /COMPONENTS=main,x64,i386" ; Check-Output $?
Start-Process -FilePath R-win.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /DIR=$env:R_LIB_PATH/R /COMPONENTS=main,x64,i386" ; Assert-Output $?
Write-Output "Done installing R"
Write-Output "Installing Rtools"
Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /SUPPRESSMSGBOXES /DIR=$RTOOLS_INSTALL_PATH" ; Check-Output $?
Start-Process -FilePath Rtools.exe -NoNewWindow -Wait -ArgumentList "/VERYSILENT /SUPPRESSMSGBOXES /DIR=$RTOOLS_INSTALL_PATH" ; Assert-Output $?
Write-Output "Done installing Rtools"
Write-Output "Installing CMake"
Add-Type -AssemblyName System.IO.Compression.FileSystem
[System.IO.Compression.ZipFile]::ExtractToDirectory("$env:CMAKE_PATH/cmake.zip", "$env:CMAKE_PATH") ; Check-Output $?
[System.IO.Compression.ZipFile]::ExtractToDirectory("$env:CMAKE_PATH/cmake.zip", "$env:CMAKE_PATH") ; Assert-Output $?
# Remove old CMake shiped with RTools
Remove-Item "$env:RTOOLS_MINGW_BIN/cmake.exe" -Force -ErrorAction Ignore
Write-Output "Done installing CMake"
Write-Output "Installing dependencies"
$packages = "c('data.table', 'jsonlite', 'knitr', 'markdown', 'Matrix', 'processx', 'R6', 'RhpcBLASctl', 'testthat'), dependencies = c('Imports', 'Depends', 'LinkingTo')"
Run-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())" ; Check-Output $?
Invoke-R-Code-Redirect-Stderr "options(install.packages.check.source = 'no'); install.packages($packages, repos = '$env:CRAN_MIRROR', type = 'binary', lib = '$env:R_LIB_PATH', Ncpus = parallel::detectCores())" ; Assert-Output $?
Write-Output "Building R-package"
......@@ -163,9 +166,9 @@ if ($env:COMPILER -ne "MSVC") {
$env:BUILD_R_FLAGS = "'--skip-install'"
} else {
Write-Output "[ERROR] Unrecognized toolchain: $env:TOOLCHAIN"
Check-Output $false
Assert-Output $false
}
Run-R-Code-Redirect-Stderr "commandArgs <- function(...){$env:BUILD_R_FLAGS}; source('build_r.R')"; Check-Output $?
Invoke-R-Code-Redirect-Stderr "commandArgs <- function(...){$env:BUILD_R_FLAGS}; source('build_r.R')"; Assert-Output $?
} elseif ($env:R_BUILD_TYPE -eq "cran") {
# NOTE: gzip and tar are needed to create a CRAN package on Windows, but
# some flavors of tar.exe can fail in some settings on Windows.
......@@ -174,7 +177,7 @@ if ($env:COMPILER -ne "MSVC") {
if ($env:R_MAJOR_VERSION -eq "3") {
$env:PATH = "C:\msys64\usr\bin;" + $env:PATH
}
Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'sh', args = 'build-cran-package.sh', echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; Check-Output $?
Invoke-R-Code-Redirect-Stderr "result <- processx::run(command = 'sh', args = 'build-cran-package.sh', echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; Assert-Output $?
Remove-From-Path ".*msys64.*"
# Test CRAN source .tar.gz in a directory that is not this repo or below it.
# When people install.packages('lightgbm'), they won't have the LightGBM
......@@ -193,31 +196,31 @@ if ($env:COMPILER -ne "MSVC") {
} else {
$check_args = "c('CMD', 'check', '--no-multiarch', '--as-cran', '--run-donttest', '$PKG_FILE_NAME')"
}
Run-R-Code-Redirect-Stderr "result <- processx::run(command = 'R.exe', args = $check_args, echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; $check_succeeded = $?
Invoke-R-Code-Redirect-Stderr "result <- processx::run(command = 'R.exe', args = $check_args, echo = TRUE, windows_verbatim_args = FALSE, error_on_status = TRUE)" ; $check_succeeded = $?
Write-Output "R CMD check build logs:"
$INSTALL_LOG_FILE_NAME = "lightgbm.Rcheck\00install.out"
Get-Content -Path "$INSTALL_LOG_FILE_NAME"
Check-Output $check_succeeded
Assert-Output $check_succeeded
Write-Output "Looking for issues with R CMD check results"
if (Get-Content "$LOG_FILE_NAME" | Select-String -Pattern "NOTE|WARNING|ERROR" -CaseSensitive -Quiet) {
echo "NOTEs, WARNINGs, or ERRORs have been found by R CMD check"
Check-Output $False
Assert-Output $False
}
} else {
$INSTALL_LOG_FILE_NAME = "$env:BUILD_SOURCESDIRECTORY\00install_out.txt"
Run-R-Code-Redirect-Stderr "source('build_r.R')" 1> $INSTALL_LOG_FILE_NAME ; $install_succeeded = $?
Invoke-R-Code-Redirect-Stderr "source('build_r.R')" 1> $INSTALL_LOG_FILE_NAME ; $install_succeeded = $?
Write-Output "----- build and install logs -----"
Get-Content -Path "$INSTALL_LOG_FILE_NAME"
Write-Output "----- end of build and install logs -----"
Check-Output $install_succeeded
Assert-Output $install_succeeded
# some errors are not raised above, but can be found in the logs
if (Get-Content "$INSTALL_LOG_FILE_NAME" | Select-String -Pattern "ERROR" -CaseSensitive -Quiet) {
echo "ERRORs have been found installing lightgbm"
Check-Output $False
Assert-Output $False
}
}
......@@ -231,7 +234,7 @@ if ($env:TOOLCHAIN -ne "MSVC") {
}
if ($checks_cnt -eq 0) {
Write-Output "Wrong R version was found (expected '$env:R_WINDOWS_VERSION'). Check the build logs."
Check-Output $False
Assert-Output $False
}
# Checking that we actually got the expected compiler. The R-package has some logic
......@@ -241,7 +244,7 @@ if ($env:R_BUILD_TYPE -eq "cmake") {
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "Check for working CXX compiler.*$env:COMPILER"
if ($checks.Matches.length -eq 0) {
Write-Output "The wrong compiler was used. Check the build logs."
Check-Output $False
Assert-Output $False
}
}
......@@ -251,7 +254,7 @@ if (($env:COMPILER -eq "MINGW") -and ($env:R_BUILD_TYPE -eq "cmake")) {
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern "Trying to build with.*$env:TOOLCHAIN"
if ($checks.Matches.length -eq 0) {
Write-Output "The wrong toolchain was used. Check the build logs."
Check-Output $False
Assert-Output $False
}
}
......@@ -267,7 +270,7 @@ if ($env:R_BUILD_TYPE -eq "cran") {
}
if ($checks_cnt -eq 0) {
Write-Output "MM_PREFETCH preprocessor definition wasn't used. Check the build logs."
Check-Output $False
Assert-Output $False
}
# Checking that MM_MALLOC preprocessor definition is actually used in CI builds.
......@@ -282,7 +285,7 @@ if ($env:R_BUILD_TYPE -eq "cran") {
}
if ($checks_cnt -eq 0) {
Write-Output "MM_MALLOC preprocessor definition wasn't used. Check the build logs."
Check-Output $False
Assert-Output $False
}
# Checking that OpenMP is actually used in CMake builds.
......@@ -290,17 +293,17 @@ if ($env:R_BUILD_TYPE -eq "cmake") {
$checks = Select-String -Path "${INSTALL_LOG_FILE_NAME}" -Pattern ".*Found OpenMP: TRUE.*"
if ($checks.Matches.length -eq 0) {
Write-Output "OpenMP wasn't found. Check the build logs."
Check-Output $False
Assert-Output $False
}
}
if ($env:COMPILER -eq "MSVC") {
Write-Output "Running tests with testthat.R"
cd R-package/tests
# NOTE: using Rscript.exe intentionally here, instead of Run-R-Code-Redirect-Stderr,
# because something about the interaction between Run-R-Code-Redirect-Stderr
# NOTE: using Rscript.exe intentionally here, instead of Invoke-R-Code-Redirect-Stderr,
# because something about the interaction between Invoke-R-Code-Redirect-Stderr
# and testthat results in failing tests not exiting with a non-0 exit code.
Rscript.exe --vanilla "testthat.R" ; Check-Output $?
Rscript.exe --vanilla "testthat.R" ; Assert-Output $?
}
Write-Output "No issues were found checking the R-package"
function Check-Output {
param( [bool]$success )
if (!$success) {
function Assert-Output {
param( [Parameter(Mandatory = $true)][bool]$success )
if (-not $success) {
$host.SetShouldExit(-1)
exit 1
}
......@@ -17,14 +17,14 @@ Remove-Item $env:TMPDIR -Force -Recurse -ErrorAction Ignore
[Void][System.IO.Directory]::CreateDirectory($env:TMPDIR)
if ($env:TASK -eq "r-package") {
& .\.ci\test-r-package-windows.ps1 ; Check-Output $?
& .\.ci\test-r-package-windows.ps1 ; Assert-Output $?
Exit 0
}
if ($env:TASK -eq "cpp-tests") {
cmake -B build -S . -DBUILD_CPP_TEST=ON -DUSE_DEBUG=ON -A x64
cmake --build build --target testlightgbm --config Debug ; Check-Output $?
.\Debug\testlightgbm.exe ; Check-Output $?
cmake --build build --target testlightgbm --config Debug ; Assert-Output $?
.\Debug\testlightgbm.exe ; Assert-Output $?
Exit 0
}
......@@ -33,23 +33,23 @@ if ($env:TASK -eq "swig") {
$ProgressPreference = "SilentlyContinue" # progress bar bug extremely slows down download speed
Invoke-WebRequest -Uri "https://sourceforge.net/projects/swig/files/latest/download" -OutFile $env:BUILD_SOURCESDIRECTORY/swig/swigwin.zip -UserAgent "curl"
Add-Type -AssemblyName System.IO.Compression.FileSystem
[System.IO.Compression.ZipFile]::ExtractToDirectory("$env:BUILD_SOURCESDIRECTORY/swig/swigwin.zip", "$env:BUILD_SOURCESDIRECTORY/swig") ; Check-Output $?
[System.IO.Compression.ZipFile]::ExtractToDirectory("$env:BUILD_SOURCESDIRECTORY/swig/swigwin.zip", "$env:BUILD_SOURCESDIRECTORY/swig") ; Assert-Output $?
$SwigFolder = Get-ChildItem -Directory -Name -Path "$env:BUILD_SOURCESDIRECTORY/swig"
$env:PATH = "$env:BUILD_SOURCESDIRECTORY/swig/$SwigFolder;" + $env:PATH
$BuildLogFileName = "$env:BUILD_SOURCESDIRECTORY\cmake_build.log"
cmake -B build -S . -A x64 -DUSE_SWIG=ON *> "$BuildLogFileName" ; $build_succeeded = $?
Write-Output "CMake build logs:"
Get-Content -Path "$BuildLogFileName"
Check-Output $build_succeeded
Assert-Output $build_succeeded
$checks = Select-String -Path "${BuildLogFileName}" -Pattern "-- Found SWIG.*${SwigFolder}/swig.exe"
$checks_cnt = $checks.Matches.length
if ($checks_cnt -eq 0) {
Write-Output "Wrong SWIG version was found (expected '${SwigFolder}'). Check the build logs."
Check-Output $False
Assert-Output $False
}
cmake --build build --target ALL_BUILD --config Release ; Check-Output $?
cmake --build build --target ALL_BUILD --config Release ; Assert-Output $?
if ($env:AZURE -eq "true") {
cp ./build/lightgbmlib.jar $env:BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_win.jar ; Check-Output $?
cp ./build/lightgbmlib.jar $env:BUILD_ARTIFACTSTAGINGDIRECTORY/lightgbmlib_win.jar ; Assert-Output $?
}
Exit 0
}
......@@ -72,7 +72,7 @@ conda create `
-y `
-n $env:CONDA_ENV `
--file $env:CONDA_REQUIREMENT_FILE `
"python=$env:PYTHON_VERSION[build=*cpython]" ; Check-Output $?
"python=$env:PYTHON_VERSION[build=*cpython]" ; Assert-Output $?
if ($env:TASK -ne "bdist") {
conda activate $env:CONDA_ENV
......@@ -80,37 +80,37 @@ if ($env:TASK -ne "bdist") {
cd $env:BUILD_SOURCESDIRECTORY
if ($env:TASK -eq "regular") {
cmake -B build -S . -A x64 ; Check-Output $?
cmake --build build --target ALL_BUILD --config Release ; Check-Output $?
sh ./build-python.sh install --precompile ; Check-Output $?
cmake -B build -S . -A x64 ; Assert-Output $?
cmake --build build --target ALL_BUILD --config Release ; Assert-Output $?
sh ./build-python.sh install --precompile ; Assert-Output $?
cp ./Release/lib_lightgbm.dll $env:BUILD_ARTIFACTSTAGINGDIRECTORY
cp ./Release/lightgbm.exe $env:BUILD_ARTIFACTSTAGINGDIRECTORY
}
elseif ($env:TASK -eq "sdist") {
sh ./build-python.sh sdist ; Check-Output $?
sh ./.ci/check-python-dists.sh ./dist ; Check-Output $?
cd dist; pip install @(Get-ChildItem *.gz) -v ; Check-Output $?
sh ./build-python.sh sdist ; Assert-Output $?
sh ./.ci/check-python-dists.sh ./dist ; Assert-Output $?
cd dist; pip install @(Get-ChildItem *.gz) -v ; Assert-Output $?
}
elseif ($env:TASK -eq "bdist") {
# Import the Chocolatey profile module so that the RefreshEnv command
# invoked below properly updates the current PowerShell session environment.
$module = "$env:ChocolateyInstall\helpers\chocolateyProfile.psm1"
Import-Module "$module" ; Check-Output $?
Import-Module "$module" ; Assert-Output $?
RefreshEnv
Write-Output "Current OpenCL drivers:"
Get-ItemProperty -Path Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Khronos\OpenCL\Vendors
conda activate $env:CONDA_ENV
sh "build-python.sh" bdist_wheel --integrated-opencl ; Check-Output $?
sh ./.ci/check-python-dists.sh ./dist ; Check-Output $?
cd dist; pip install @(Get-ChildItem *py3-none-win_amd64.whl) ; Check-Output $?
sh "build-python.sh" bdist_wheel --integrated-opencl ; Assert-Output $?
sh ./.ci/check-python-dists.sh ./dist ; Assert-Output $?
cd dist; pip install @(Get-ChildItem *py3-none-win_amd64.whl) ; Assert-Output $?
cp @(Get-ChildItem *py3-none-win_amd64.whl) $env:BUILD_ARTIFACTSTAGINGDIRECTORY
} elseif (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python")) {
if ($env:COMPILER -eq "MINGW") {
sh ./build-python.sh install --mingw ; Check-Output $?
sh ./build-python.sh install --mingw ; Assert-Output $?
} else {
sh ./build-python.sh install; Check-Output $?
sh ./build-python.sh install; Assert-Output $?
}
}
......@@ -125,7 +125,7 @@ if ($env:TASK -eq "bdist") {
$env:LIGHTGBM_TEST_DUAL_CPU_GPU = "1"
}
pytest $tests ; Check-Output $?
pytest $tests ; Assert-Output $?
if (($env:TASK -eq "regular") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -eq "python"))) {
cd $env:BUILD_SOURCESDIRECTORY/examples/python-guide
......@@ -134,9 +134,9 @@ if (($env:TASK -eq "regular") -or (($env:APPVEYOR -eq "true") -and ($env:TASK -e
conda install -y -n $env:CONDA_ENV "h5py>=3.10" "ipywidgets>=8.1.2" "notebook>=7.1.2"
foreach ($file in @(Get-ChildItem *.py)) {
@("import sys, warnings", "warnings.showwarning = lambda message, category, filename, lineno, file=None, line=None: sys.stdout.write(warnings.formatwarning(message, category, filename, lineno, line))") + (Get-Content $file) | Set-Content $file
python $file ; Check-Output $?
python $file ; Assert-Output $?
} # run all examples
cd $env:BUILD_SOURCESDIRECTORY/examples/python-guide/notebooks
(Get-Content "interactive_plot_example.ipynb").replace('INTERACTIVE = False', 'assert False, \"Interactive mode disabled\"') | Set-Content "interactive_plot_example.ipynb"
jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb ; Check-Output $? # run all notebooks
jupyter nbconvert --ExecutePreprocessor.timeout=180 --to notebook --execute --inplace *.ipynb ; Assert-Output $? # run all notebooks
}
......@@ -98,6 +98,9 @@ if [[ $TASK == "swig" ]]; then
fi
if [[ $TASK == "lint" ]]; then
pwsh -command "Install-Module -Name PSScriptAnalyzer -Scope CurrentUser -SkipPublisherCheck"
echo "Linting PowerShell code"
pwsh -file "./.ci/lint-powershell.ps1" || exit 0
conda create -q -y -n "${CONDA_ENV}" \
"${CONDA_PYTHON_REQUIREMENT}" \
'cmakelint>=1.4.3' \
......
......@@ -7,7 +7,7 @@ insert_final_newline = true
indent_style = space
indent_size = 2
[*.{py,sh,js}]
[*.{py,sh,js,ps1}]
indent_size = 4
line_length = 120
skip = external_libs
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment