[ci] Remove temp workaround for graphviz installation (#5126)

* Update test_windows.ps1

* Update .appveyor.yml

* Update .appveyor.yml

.ci/test_windows.ps1

@@ -52,7 +52,7 @@ if ($env:TASK -eq "swig") {
 
 conda install -q -y -n $env:CONDA_ENV cloudpickle joblib matplotlib numpy pandas psutil pytest scikit-learn scipy ; Check-Output $?
 # python-graphviz has to be installed separately to prevent conda from downgrading to pypy
-conda install -q -y -n $env:CONDA_ENV libxml2 python-graphviz ; Check-Output $?
+conda install -q -y -n $env:CONDA_ENV python-graphviz ; Check-Output $?
 
 if ($env:TASK -eq "regular") {
   mkdir $env:BUILD_SOURCESDIRECTORY/build; cd $env:BUILD_SOURCESDIRECTORY/build