[ci] Prevent Python downgrading to pypy on Windows (2) (#5198)

* Update test_windows.ps1

* Update .appveyor.yml

* Update test_windows.ps1

* Update test_windows.ps1

* Update test_windows.ps1

* Update test_windows.ps1

* Update test_windows.ps1

* Update .appveyor.yml

.ci/test_windows.ps1

@@ -50,9 +50,9 @@ if ($env:TASK -eq "swig") {
   Exit 0
 }
 
-conda install -q -y -n $env:CONDA_ENV cloudpickle joblib matplotlib numpy pandas psutil pytest scikit-learn scipy ; Check-Output $?
-# python-graphviz has to be installed separately to prevent conda from downgrading to pypy
-conda install -q -y -n $env:CONDA_ENV libxml2 python-graphviz ; Check-Output $?
+conda install -q -y -n $env:CONDA_ENV cloudpickle joblib numpy pandas psutil pytest scikit-learn scipy ; Check-Output $?
+# matplotlib and python-graphviz have to be installed separately to prevent conda from downgrading to pypy
+conda install -q -y -n $env:CONDA_ENV matplotlib python-graphviz ; Check-Output $?
 
 if ($env:TASK -eq "regular") {
   mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build