Unverified Commit df8d7767 authored by peterjc123's avatar peterjc123 Committed by GitHub
Browse files

Reuse common logic and migrate CUDA >= 10.1 jobs to VS 2019 for Windo… (#2264)

parent a85f21d2
......@@ -6,14 +6,17 @@ version: 2.1
# - Replace binary_linux_wheel_py3.7 with the name of the job you want to test.
# Job names are 'name:' key.
orbs:
win: circleci/windows@2.0.0
executors:
windows-gpu-prototype:
windows-cpu:
machine:
resource_class: windows.gpu.small.prototype
image: windows-server-2019-nvidia:201908-28
resource_class: windows.xlarge
image: windows-server-2019-vs2019:stable
shell: bash.exe
windows-gpu:
machine:
resource_class: windows.gpu.nvidia.medium
image: windows-server-2019-nvidia:stable
shell: bash.exe
commands:
......@@ -75,31 +78,6 @@ binary_common: &binary_common
UNICODE_ABI: << parameters.unicode_abi >>
CU_VERSION: << parameters.cu_version >>
binary_windows: &binary_windows
parameters:
# Edit these defaults to do a release`
build_version:
description: "version number of release binary; by default, build a nightly"
type: string
default: ""
pytorch_version:
description: "PyTorch version to build against; by default, use a nightly"
type: string
default: ""
# Don't edit these
python_version:
description: "Python version to build against (e.g., 3.7)"
type: string
cu_version:
description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
type: string
environment:
DESIRED_PYTHON: << parameters.python_version >>
PYTORCH_VERSION: << parameters.pytorch_version >>
CUDA_VERSION: << parameters.cu_version >>
USE_SCCACHE: "1"
VC_YEAR: "2017"
jobs:
circleci_consistency:
docker:
......@@ -248,71 +226,75 @@ jobs:
binary_win_conda:
<<: *binary_common
executor:
name: win/default
shell: bash.exe
executor: windows-cpu
steps:
- checkout_merge
- run:
command: |
choco install miniconda3
(& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression
set -ex
source packaging/windows/internal/vc_install_helper.sh
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
conda install -yq conda-build "conda-package-handling!=1.5.0"
bash packaging/build_conda.sh
shell: powershell.exe
packaging/build_conda.sh
- store_test_results:
path: build_results/
binary_win_conda_cuda:
<<: *binary_common
executor: windows-gpu-prototype
executor: windows-gpu
steps:
- checkout_merge
- run:
command: |
choco install miniconda3
(& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression
set -ex
source packaging/windows/internal/vc_install_helper.sh
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
conda install -yq conda-build "conda-package-handling!=1.5.0"
bash packaging/build_conda.sh
shell: powershell.exe
packaging/build_conda.sh
binary_win_conda_release:
<<: *binary_windows
executor:
name: win/default
shell: cmd.exe
<<: *binary_common
executor: windows-cpu
steps:
- checkout_merge
- run:
name: Build conda packages
command: |
call packaging/windows/internal/build_conda.bat
set -ex
source packaging/windows/internal/vc_install_helper.sh
packaging/windows/internal/cuda_install.bat
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
conda install -yq conda-build "conda-package-handling!=1.5.0"
packaging/build_conda.sh
rm /C/tools/miniconda3/conda-bld/win-64/vs2019*.tar.bz2
- store_artifacts:
path: packaging/windows/output
path: C:/tools/miniconda3/conda-bld/win-64
- persist_to_workspace:
root: packaging/windows/output
root: C:/tools/miniconda3/conda-bld/win-64
paths:
- "*"
- store_test_results:
path: build_results/
binary_win_wheel_release:
<<: *binary_windows
executor:
name: win/default
shell: cmd.exe
<<: *binary_common
executor: windows-cpu
steps:
- checkout_merge
- run:
name: Build wheel packages
command: |
call packaging/windows/internal/build_wheels.bat
set -ex
source packaging/windows/internal/vc_install_helper.sh
packaging/windows/internal/cuda_install.bat
packaging/build_wheel.sh
- store_artifacts:
path: packaging/windows/output
path: dist
- persist_to_workspace:
root: packaging/windows/output
root: dist
paths:
- "*"
- store_test_results:
......@@ -491,21 +473,21 @@ workflows:
name: binary_win_wheel_py3.6_cpu
python_version: '3.6'
- binary_win_wheel_release:
cu_version: '92'
cu_version: cu92
filters:
branches:
only: master
name: binary_win_wheel_py3.6_cu92
python_version: '3.6'
- binary_win_wheel_release:
cu_version: '101'
cu_version: cu101
filters:
branches:
only: master
name: binary_win_wheel_py3.6_cu101
python_version: '3.6'
- binary_win_wheel_release:
cu_version: '102'
cu_version: cu102
filters:
branches:
only: master
......@@ -519,21 +501,21 @@ workflows:
name: binary_win_wheel_py3.7_cpu
python_version: '3.7'
- binary_win_wheel_release:
cu_version: '92'
cu_version: cu92
filters:
branches:
only: master
name: binary_win_wheel_py3.7_cu92
python_version: '3.7'
- binary_win_wheel_release:
cu_version: '101'
cu_version: cu101
filters:
branches:
only: master
name: binary_win_wheel_py3.7_cu101
python_version: '3.7'
- binary_win_wheel_release:
cu_version: '102'
cu_version: cu102
filters:
branches:
only: master
......@@ -544,21 +526,21 @@ workflows:
name: binary_win_wheel_py3.8_cpu
python_version: '3.8'
- binary_win_wheel_release:
cu_version: '92'
cu_version: cu92
filters:
branches:
only: master
name: binary_win_wheel_py3.8_cu92
python_version: '3.8'
- binary_win_wheel_release:
cu_version: '101'
cu_version: cu101
filters:
branches:
only: master
name: binary_win_wheel_py3.8_cu101
python_version: '3.8'
- binary_win_wheel_release:
cu_version: '102'
cu_version: cu102
name: binary_win_wheel_py3.8_cu102
python_version: '3.8'
- binary_linux_conda:
......@@ -644,21 +626,21 @@ workflows:
name: binary_win_conda_py3.6_cpu
python_version: '3.6'
- binary_win_conda_release:
cu_version: '92'
cu_version: cu92
filters:
branches:
only: master
name: binary_win_conda_py3.6_cu92
python_version: '3.6'
- binary_win_conda_release:
cu_version: '101'
cu_version: cu101
filters:
branches:
only: master
name: binary_win_conda_py3.6_cu101
python_version: '3.6'
- binary_win_conda_release:
cu_version: '102'
cu_version: cu102
filters:
branches:
only: master
......@@ -672,21 +654,21 @@ workflows:
name: binary_win_conda_py3.7_cpu
python_version: '3.7'
- binary_win_conda_release:
cu_version: '92'
cu_version: cu92
filters:
branches:
only: master
name: binary_win_conda_py3.7_cu92
python_version: '3.7'
- binary_win_conda_release:
cu_version: '101'
cu_version: cu101
filters:
branches:
only: master
name: binary_win_conda_py3.7_cu101
python_version: '3.7'
- binary_win_conda_release:
cu_version: '102'
cu_version: cu102
filters:
branches:
only: master
......@@ -697,21 +679,21 @@ workflows:
name: binary_win_conda_py3.8_cpu
python_version: '3.8'
- binary_win_conda_release:
cu_version: '92'
cu_version: cu92
filters:
branches:
only: master
name: binary_win_conda_py3.8_cu92
python_version: '3.8'
- binary_win_conda_release:
cu_version: '101'
cu_version: cu101
filters:
branches:
only: master
name: binary_win_conda_py3.8_cu101
python_version: '3.8'
- binary_win_conda_release:
cu_version: '102'
cu_version: cu102
name: binary_win_conda_py3.8_cu102
python_version: '3.8'
- binary_linux_conda_cuda:
......@@ -1040,7 +1022,7 @@ workflows:
- nightly_binary_win_wheel_py3.6_cpu
subfolder: cpu/
- binary_win_wheel_release:
cu_version: '92'
cu_version: cu92
filters:
branches:
only: nightly
......@@ -1058,7 +1040,7 @@ workflows:
- nightly_binary_win_wheel_py3.6_cu92
subfolder: cu92/
- binary_win_wheel_release:
cu_version: '101'
cu_version: cu101
filters:
branches:
only: nightly
......@@ -1076,7 +1058,7 @@ workflows:
- nightly_binary_win_wheel_py3.6_cu101
subfolder: cu101/
- binary_win_wheel_release:
cu_version: '102'
cu_version: cu102
filters:
branches:
only: nightly
......@@ -1112,7 +1094,7 @@ workflows:
- nightly_binary_win_wheel_py3.7_cpu
subfolder: cpu/
- binary_win_wheel_release:
cu_version: '92'
cu_version: cu92
filters:
branches:
only: nightly
......@@ -1130,7 +1112,7 @@ workflows:
- nightly_binary_win_wheel_py3.7_cu92
subfolder: cu92/
- binary_win_wheel_release:
cu_version: '101'
cu_version: cu101
filters:
branches:
only: nightly
......@@ -1148,7 +1130,7 @@ workflows:
- nightly_binary_win_wheel_py3.7_cu101
subfolder: cu101/
- binary_win_wheel_release:
cu_version: '102'
cu_version: cu102
filters:
branches:
only: nightly
......@@ -1184,7 +1166,7 @@ workflows:
- nightly_binary_win_wheel_py3.8_cpu
subfolder: cpu/
- binary_win_wheel_release:
cu_version: '92'
cu_version: cu92
filters:
branches:
only: nightly
......@@ -1202,7 +1184,7 @@ workflows:
- nightly_binary_win_wheel_py3.8_cu92
subfolder: cu92/
- binary_win_wheel_release:
cu_version: '101'
cu_version: cu101
filters:
branches:
only: nightly
......@@ -1220,7 +1202,7 @@ workflows:
- nightly_binary_win_wheel_py3.8_cu101
subfolder: cu101/
- binary_win_wheel_release:
cu_version: '102'
cu_version: cu102
filters:
branches:
only: nightly
......@@ -1525,7 +1507,7 @@ workflows:
requires:
- nightly_binary_win_conda_py3.6_cpu
- binary_win_conda_release:
cu_version: '92'
cu_version: cu92
filters:
branches:
only: nightly
......@@ -1542,7 +1524,7 @@ workflows:
requires:
- nightly_binary_win_conda_py3.6_cu92
- binary_win_conda_release:
cu_version: '101'
cu_version: cu101
filters:
branches:
only: nightly
......@@ -1559,7 +1541,7 @@ workflows:
requires:
- nightly_binary_win_conda_py3.6_cu101
- binary_win_conda_release:
cu_version: '102'
cu_version: cu102
filters:
branches:
only: nightly
......@@ -1593,7 +1575,7 @@ workflows:
requires:
- nightly_binary_win_conda_py3.7_cpu
- binary_win_conda_release:
cu_version: '92'
cu_version: cu92
filters:
branches:
only: nightly
......@@ -1610,7 +1592,7 @@ workflows:
requires:
- nightly_binary_win_conda_py3.7_cu92
- binary_win_conda_release:
cu_version: '101'
cu_version: cu101
filters:
branches:
only: nightly
......@@ -1627,7 +1609,7 @@ workflows:
requires:
- nightly_binary_win_conda_py3.7_cu101
- binary_win_conda_release:
cu_version: '102'
cu_version: cu102
filters:
branches:
only: nightly
......@@ -1661,7 +1643,7 @@ workflows:
requires:
- nightly_binary_win_conda_py3.8_cpu
- binary_win_conda_release:
cu_version: '92'
cu_version: cu92
filters:
branches:
only: nightly
......@@ -1678,7 +1660,7 @@ workflows:
requires:
- nightly_binary_win_conda_py3.8_cu92
- binary_win_conda_release:
cu_version: '101'
cu_version: cu101
filters:
branches:
only: nightly
......@@ -1695,7 +1677,7 @@ workflows:
requires:
- nightly_binary_win_conda_py3.8_cu101
- binary_win_conda_release:
cu_version: '102'
cu_version: cu102
filters:
branches:
only: nightly
......
......@@ -6,14 +6,17 @@ version: 2.1
# - Replace binary_linux_wheel_py3.7 with the name of the job you want to test.
# Job names are 'name:' key.
orbs:
win: circleci/windows@2.0.0
executors:
windows-gpu-prototype:
windows-cpu:
machine:
resource_class: windows.gpu.small.prototype
image: windows-server-2019-nvidia:201908-28
resource_class: windows.xlarge
image: windows-server-2019-vs2019:stable
shell: bash.exe
windows-gpu:
machine:
resource_class: windows.gpu.nvidia.medium
image: windows-server-2019-nvidia:stable
shell: bash.exe
commands:
......@@ -75,31 +78,6 @@ binary_common: &binary_common
UNICODE_ABI: << parameters.unicode_abi >>
CU_VERSION: << parameters.cu_version >>
binary_windows: &binary_windows
parameters:
# Edit these defaults to do a release`
build_version:
description: "version number of release binary; by default, build a nightly"
type: string
default: ""
pytorch_version:
description: "PyTorch version to build against; by default, use a nightly"
type: string
default: ""
# Don't edit these
python_version:
description: "Python version to build against (e.g., 3.7)"
type: string
cu_version:
description: "CUDA version to build against, in CU format (e.g., cpu or cu100)"
type: string
environment:
DESIRED_PYTHON: << parameters.python_version >>
PYTORCH_VERSION: << parameters.pytorch_version >>
CUDA_VERSION: << parameters.cu_version >>
USE_SCCACHE: "1"
VC_YEAR: "2017"
jobs:
circleci_consistency:
docker:
......@@ -248,71 +226,75 @@ jobs:
binary_win_conda:
<<: *binary_common
executor:
name: win/default
shell: bash.exe
executor: windows-cpu
steps:
- checkout_merge
- run:
command: |
choco install miniconda3
(& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression
set -ex
source packaging/windows/internal/vc_install_helper.sh
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
conda install -yq conda-build "conda-package-handling!=1.5.0"
bash packaging/build_conda.sh
shell: powershell.exe
packaging/build_conda.sh
- store_test_results:
path: build_results/
binary_win_conda_cuda:
<<: *binary_common
executor: windows-gpu-prototype
executor: windows-gpu
steps:
- checkout_merge
- run:
command: |
choco install miniconda3
(& "C:\tools\miniconda3\Scripts\conda.exe" "shell.powershell" "hook") | Out-String | Invoke-Expression
set -ex
source packaging/windows/internal/vc_install_helper.sh
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
conda install -yq conda-build "conda-package-handling!=1.5.0"
bash packaging/build_conda.sh
shell: powershell.exe
packaging/build_conda.sh
binary_win_conda_release:
<<: *binary_windows
executor:
name: win/default
shell: cmd.exe
<<: *binary_common
executor: windows-cpu
steps:
- checkout_merge
- run:
name: Build conda packages
command: |
call packaging/windows/internal/build_conda.bat
set -ex
source packaging/windows/internal/vc_install_helper.sh
packaging/windows/internal/cuda_install.bat
eval "$('/C/tools/miniconda3/Scripts/conda.exe' 'shell.bash' 'hook')"
conda activate base
conda install -yq conda-build "conda-package-handling!=1.5.0"
packaging/build_conda.sh
rm /C/tools/miniconda3/conda-bld/win-64/vs2019*.tar.bz2
- store_artifacts:
path: packaging/windows/output
path: C:/tools/miniconda3/conda-bld/win-64
- persist_to_workspace:
root: packaging/windows/output
root: C:/tools/miniconda3/conda-bld/win-64
paths:
- "*"
- store_test_results:
path: build_results/
binary_win_wheel_release:
<<: *binary_windows
executor:
name: win/default
shell: cmd.exe
<<: *binary_common
executor: windows-cpu
steps:
- checkout_merge
- run:
name: Build wheel packages
command: |
call packaging/windows/internal/build_wheels.bat
set -ex
source packaging/windows/internal/vc_install_helper.sh
packaging/windows/internal/cuda_install.bat
packaging/build_wheel.sh
- store_artifacts:
path: packaging/windows/output
path: dist
- persist_to_workspace:
root: packaging/windows/output
root: dist
paths:
- "*"
- store_test_results:
......
......@@ -76,7 +76,7 @@ def generate_base_workflow(base_workflow_name, python_version, cu_version,
d = {
"name": base_workflow_name,
"python_version": python_version,
"cu_version": cu_version.replace("cu", "") if os_type == "win" else cu_version,
"cu_version": cu_version,
}
if os_type != "win" and unicode:
......
......@@ -10,4 +10,8 @@ setup_wheel_python
pip_install numpy pyyaml future ninja
setup_pip_pytorch_version
python setup.py clean
IS_WHEEL=1 python setup.py bdist_wheel
if [[ "$OSTYPE" == "msys" ]]; then
IS_WHEEL=1 "$script_dir/windows/internal/vc_env_helper.bat" python setup.py bdist_wheel
else
IS_WHEEL=1 python setup.py bdist_wheel
fi
......@@ -165,7 +165,7 @@ retry () {
#
# Precondition: If Linux, you are in a soumith/manylinux-cuda* Docker image
setup_wheel_python() {
if [[ "$(uname)" == Darwin ]]; then
if [[ "$(uname)" == Darwin || "$OSTYPE" == "msys" ]]; then
eval "$(conda shell.bash hook)"
conda env remove -n "env$PYTHON_VERSION" || true
conda create -yn "env$PYTHON_VERSION" python="$PYTHON_VERSION"
......@@ -285,8 +285,7 @@ setup_conda_cudatoolkit_constraint() {
# Build the proper compiler package before building the final package
setup_visual_studio_constraint() {
if [[ "$OSTYPE" == "msys" ]]; then
export VSTOOLCHAIN_PACKAGE=vs2019
export VSDEVCMD_ARGS=''
export VSTOOLCHAIN_PACKAGE=vs$VC_YEAR
conda build $CONDA_CHANNEL_FLAGS --no-anaconda-upload packaging/$VSTOOLCHAIN_PACKAGE
cp packaging/$VSTOOLCHAIN_PACKAGE/conda_build_config.yaml packaging/torchvision/conda_build_config.yaml
fi
......
......@@ -19,27 +19,6 @@ outputs:
# VS 2019 is binary-compatible with VS 2017/vc 14.1 and 2015/vc14. Tools are "v142".
strong:
- vc{{ vcfeature }}
run_exports:
- vc {{ vcver }}
about:
summary: Activation and version verification of MSVC {{ vcver }} (VS {{ vsyear }}) compiler
license: BSD 3-clause
- name: vs{{ vsyear }}_runtime
script: install_runtime.bat
- name: vc
version: {{ vcver }}
track_features:
- vc{{ vcfeature }}
requirements:
run:
- {{ pin_subpackage('vs' ~ vsyear ~ '_runtime') }}
about:
home: https://github.com/conda/conda/wiki/VC-features
license: Modified BSD License (3-clause)
license_family: BSD
summary: A meta-package to track VC features.
description: |
This metapackage is used to activate vc features without
depending on Python.
doc_url: https://github.com/conda/conda/wiki/VC-features
dev_url: https://github.com/conda/conda/wiki/VC-features
if "%VC_YEAR%" == "2017" set VSDEVCMD_ARGS=-vcvars_ver=14.11
if "%VC_YEAR%" == "2017" powershell packaging/windows/internal/vs_install.ps1
if "%VC_YEAR%" == "2017" powershell packaging/windows/internal/vs2017_install.ps1
if errorlevel 1 exit /b 1
call packaging/windows/internal/cuda_install.bat
......
if "%VC_YEAR%" == "2017" set VSDEVCMD_ARGS=-vcvars_ver=14.11
if "%VC_YEAR%" == "2017" powershell packaging/windows/internal/vs_install.ps1
if "%VC_YEAR%" == "2017" powershell packaging/windows/internal/vs2017_install.ps1
if errorlevel 1 exit /b 1
call packaging/windows/internal/cuda_install.bat
......
@echo on
if "%CUDA_VERSION%" == "cpu" (
if "%CU_VERSION%" == "cpu" (
echo Skipping for CPU builds
exit /b 0
)
......@@ -9,9 +9,9 @@ set SRC_DIR=%~dp0\..
if not exist "%SRC_DIR%\temp_build" mkdir "%SRC_DIR%\temp_build"
set /a CUDA_VER=%CUDA_VERSION%
set CUDA_VER_MAJOR=%CUDA_VERSION:~0,-1%
set CUDA_VER_MINOR=%CUDA_VERSION:~-1,1%
set /a CUDA_VER=%CU_VERSION:cu=%
set CUDA_VER_MAJOR=%CUDA_VER:~0,-1%
set CUDA_VER_MINOR=%CUDA_VER:~-1,1%
set CUDA_VERSION_STR=%CUDA_VER_MAJOR%.%CUDA_VER_MINOR%
if %CUDA_VER% EQU 92 goto cuda92
......
@echo on
set VC_VERSION_LOWER=16
set VC_VERSION_UPPER=17
if "%VC_YEAR%" == "2017" (
set VC_VERSION_LOWER=15
set VC_VERSION_UPPER=16
)
for /f "usebackq tokens=*" %%i in (`"%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe" -legacy -products * -version [%VC_VERSION_LOWER%^,%VC_VERSION_UPPER%^) -property installationPath`) do (
if exist "%%i" if exist "%%i\VC\Auxiliary\Build\vcvarsall.bat" (
set "VS15INSTALLDIR=%%i"
set "VS15VCVARSALL=%%i\VC\Auxiliary\Build\vcvarsall.bat"
goto vswhere
)
)
:vswhere
if "%VSDEVCMD_ARGS%" == "" (
call "%VS15VCVARSALL%" x64 || exit /b 1
) else (
call "%VS15VCVARSALL%" x64 %VSDEVCMD_ARGS% || exit /b 1
)
@echo on
set DISTUTILS_USE_SDK=1
set args=%1
shift
:start
if [%1] == [] goto done
set args=%args% %1
shift
goto start
:done
if "%args%" == "" (
echo Usage: vc_env_helper.bat [command] [args]
echo e.g. vc_env_helper.bat cl /c test.cpp
)
%args% || exit /b 1
#!/bin/bash
set -ex
if [[ "$CU_VERSION" == "cu92" ]]; then
export VC_YEAR=2017
export VSDEVCMD_ARGS="-vcvars_ver=14.11"
powershell packaging/windows/internal/vs2017_install.ps1
elif [[ "$CU_VERSION" == "cu100" ]]; then
export VC_YEAR=2017
export VSDEVCMD_ARGS=""
powershell packaging/windows/internal/vs2017_install.ps1
else
export VC_YEAR=2019
export VSDEVCMD_ARGS=""
fi
$VS_DOWNLOAD_LINK = "https://aka.ms/vs/16/release/vs_buildtools.exe"
$VS_INSTALL_ARGS = @("--nocache","--quiet","--wait", "--add Microsoft.VisualStudio.Workload.VCTools",
"--add Microsoft.Component.MSBuild",
"--add Microsoft.VisualStudio.Component.Roslyn.Compiler",
"--add Microsoft.VisualStudio.Component.VC.CoreBuildTools",
"--add Microsoft.VisualStudio.Component.VC.Redist.14.Latest",
"--add Microsoft.VisualStudio.Component.VC.Tools.x86.x64")
curl.exe --retry 3 -kL $VS_DOWNLOAD_LINK --output vs_installer.exe
if ($LASTEXITCODE -ne 0) {
echo "Download of the VS 2019 installer failed"
exit 1
}
$process = Start-Process "${PWD}\vs_installer.exe" -ArgumentList $VS_INSTALL_ARGS -NoNewWindow -Wait -PassThru
Remove-Item -Path vs_installer.exe -Force
$exitCode = $process.ExitCode
if (($exitCode -ne 0) -and ($exitCode -ne 3010)) {
echo "VS 2019 installer exited with code $exitCode, which should be one of [0, 3010]."
exit 1
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment