Skip to content

GitLab

Commit 79a3eaf2 authored by yan.yan's avatar yan.yan
Browse files

time to release!

parent b67ad2a8
Hide whitespace changes
Inline Side-by-side
Showing with 184 additions and 186 deletions
.github/workflows/build.yaml
View file @ 79a3eaf2
......@@ -11,53 +11,53 @@ on:
- '*'
jobs:
# build-windows:
# runs-on: windows-latest
# strategy:
# matrix:
# python-version: ['3.7', '3.8', '3.9', '3.10']
# cuda-version: ['10.2', '11.1', '11.4']
# steps:
# - uses: actions/checkout@master
# - name: Install CUDA
# env:
# CUDA_VERSION: ${{ matrix.cuda-version }}
# PYTHON_VERSION: ${{ matrix.python-version }}
# cuda: ${{ matrix.cuda-version }}
# if: (github.event_name == 'push' && (startsWith(github.ref, 'refs/tags')) && (env.CUDA_VERSION != '') ) || (env.CUDA_VERSION == '11.1' && env.PYTHON_VERSION == '3.10')
# shell: powershell
# run: .\tools\install_windows_cuda.ps1
# - name: Set up Python ${{ matrix.python-version }}
# uses: actions/setup-python@v2
# with:
# python-version: ${{ matrix.python-version }}
# - uses: ilammy/msvc-dev-cmd@v1
# - name: Install pep build
# run: |
# python -m pip install build --user
# python -m pip install --upgrade pip twine wheel
# python -m pip install pytest setuptools
build-windows:
runs-on: windows-latest
strategy:
matrix:
python-version: ['3.7', '3.8', '3.9', '3.10']
cuda-version: ['10.2', '11.1', '11.4']
steps:
- uses: actions/checkout@master
- name: Install CUDA
env:
CUDA_VERSION: ${{ matrix.cuda-version }}
PYTHON_VERSION: ${{ matrix.python-version }}
cuda: ${{ matrix.cuda-version }}
if: (github.event_name == 'push' && (startsWith(github.ref, 'refs/tags')) && (env.CUDA_VERSION != '') ) || (env.CUDA_VERSION == '11.1' && env.PYTHON_VERSION == '3.10')
shell: powershell
run: .\tools\install_windows_cuda.ps1
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v2
with:
python-version: ${{ matrix.python-version }}
- uses: ilammy/msvc-dev-cmd@v1
- name: Install pep build
run: |
python -m pip install build --user
python -m pip install --upgrade pip twine wheel
python -m pip install pytest setuptools
# - name: Build a windows binary wheel
# env:
# CUDA_VERSION: ${{ matrix.cuda-version }}
# PYTHON_VERSION: ${{ matrix.python-version }}
# if: (github.event_name == 'push' && (startsWith(github.ref, 'refs/tags')) ) || (env.CUDA_VERSION == '11.1' && env.PYTHON_VERSION == '3.10')
# run: |
# $Env:CUMM_CUDA_VERSION = "${{ matrix.cuda-version }}"
# $Env:CUMM_CUDA_ARCH_LIST = "all"
# $Env:SPCONV_DISABLE_JIT = "1"
# pip install pccm pybind11
# python -m build --wheel --outdir dist/ .
# shell: powershell
- name: Build a windows binary wheel
env:
CUDA_VERSION: ${{ matrix.cuda-version }}
PYTHON_VERSION: ${{ matrix.python-version }}
if: (github.event_name == 'push' && (startsWith(github.ref, 'refs/tags')) ) || (env.CUDA_VERSION == '11.1' && env.PYTHON_VERSION == '3.10')
run: |
$Env:CUMM_CUDA_VERSION = "${{ matrix.cuda-version }}"
$Env:CUMM_CUDA_ARCH_LIST = "all"
$Env:SPCONV_DISABLE_JIT = "1"
pip install pccm pybind11
python -m build --wheel --outdir dist/ .
shell: powershell
# - name: Publish a Python distribution to PyPI
# if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
# run: |
# $Env:TWINE_USERNAME = "__token__"
# $Env:TWINE_PASSWORD = "${{ secrets.pypi_password }}"
# twine upload dist/*
# shell: powershell
- name: Publish a Python distribution to PyPI
if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
run: |
$Env:TWINE_USERNAME = "__token__"
$Env:TWINE_PASSWORD = "${{ secrets.pypi_password }}"
twine upload dist/*
shell: powershell
build:
# needs: build-windows
......
README.md
View file @ 79a3eaf2
......@@ -18,8 +18,6 @@
[![Build Status](https://github.com/traveller59/spconv/workflows/build/badge.svg)](https://github.com/traveller59/spconv/actions?query=workflow%3Abuild)
# WORK IN PROGRESS, DON'T USE!!!
## Breaking changes in Spconv 2.x
* ```spconv.xxx``` move to ```spconv.pytorch.xxx```, change all ```import spconv``` to ```import spconv.pytorch as spconv``` and ```from spconv.xxx import``` to ```from spconv.pytorch.xxx import```.
......
setup.py
View file @ 79a3eaf2
......@@ -156,7 +156,7 @@ if disable_jit is not None and disable_jit == "1":
else:
std = "c++17"
ext_modules: List[Extension] = [
PCCMExtension([cu],
PCCMExtension([cu, SpconvOps()],
"spconv/core_cc",
Path(__file__).resolve().parent / "spconv",
objects_folder="objects",
......
spconv/core.py
View file @ 79a3eaf2
......@@ -32,173 +32,173 @@ class AlgoHint(Enum):
# TODO two step build: build gemm kernels first, then bind for every python
SHUFFLE_SIMT_PARAMS: List[GemmAlgoParams] = [
# *gen_shuffle_params(
# (64, 128, 32), (32, 64, 32), ["s8,s8,s32,s32,s32"],
# 2, kernel.GemmAlgo.SimtDP4A, None),
# *gen_shuffle_params(
# (128, 64, 32), (64, 32, 32), ["s8,s8,s32,s32,s32"],
# 2, kernel.GemmAlgo.SimtDP4A, None),
# *gen_shuffle_params(
# (128, 128, 32),
# (32, 64, 32), ["s8,s8,s32,s32,s32"], 2,
# kernel.GemmAlgo.SimtDP4A, None),
# *gen_shuffle_params(
# (128, 128, 32),
# (64, 32, 32), ["s8,s8,s8,s32,s32", "s8,s8,s32,s32,s32"], 2,
# kernel.GemmAlgo.SimtDP4A, None),
*gen_shuffle_params(
(64, 128, 32), (32, 64, 32), ["s8,s8,s32,s32,s32"],
2, kernel.GemmAlgo.SimtDP4A, None),
*gen_shuffle_params(
(128, 64, 32), (64, 32, 32), ["s8,s8,s32,s32,s32"],
2, kernel.GemmAlgo.SimtDP4A, None),
*gen_shuffle_params(
(128, 128, 32),
(32, 64, 32), ["s8,s8,s32,s32,s32"], 2,
kernel.GemmAlgo.SimtDP4A, None),
*gen_shuffle_params(
(128, 128, 32),
(64, 32, 32), ["s8,s8,s8,s32,s32", "s8,s8,s32,s32,s32"], 2,
kernel.GemmAlgo.SimtDP4A, None),
*gen_shuffle_params(
(64, 64, 32), (32, 32, 32), ["s8,s8,s32,s32,s32"],
2, kernel.GemmAlgo.SimtDP4A, None),
*gen_shuffle_params(
(64, 256, 8),
(32, 64, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# *gen_shuffle_params(
# (64, 256, 8),
# (32, 64, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# # *gen_shuffle_params(
# # (64, 256, 8),
# # (64, 32, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# *gen_shuffle_params(
# (32, 128, 16),
# (32, 32, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# *gen_shuffle_params(
# (32, 512, 8),
# (32, 64, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# # *gen_shuffle_params(
# # (128, 128, 8),
# # (64, 32, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# (64, 32, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
*gen_shuffle_params(
(32, 128, 16),
(32, 32, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
*gen_shuffle_params(
(32, 512, 8),
(32, 64, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# *gen_shuffle_params(
# (128, 128, 8),
# (32, 64, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# (64, 32, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
*gen_shuffle_params(
(128, 128, 8),
(32, 64, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
*gen_shuffle_params(
(64, 128, 8),
(32, 64, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# *gen_shuffle_params(
# (64, 128, 8),
# (32, 64, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# # *gen_shuffle_params(
# # (64, 128, 8),
# # (64, 32, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# # *gen_shuffle_params(
# # (128, 64, 8),
# # (32, 64, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# *gen_shuffle_params(
# (128, 64, 8),
# (64, 32, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# *gen_shuffle_params(
# (64, 64, 8),
# (32, 32, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# *gen_shuffle_params(
# (32, 64, 16),
# (32, 32, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# *gen_shuffle_params(
# (64, 32, 16),
# (32, 32, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# *gen_shuffle_params(
# (32, 32, 32),
# (32, 32, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# # fall back kernels if mat is misaligned for half
# # *gen_shuffle_params(
# # (128, 128, 8),
# # (32, 64, 8), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# *gen_shuffle_params(
# (32, 64, 32),
# (32, 32, 8), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# *gen_shuffle_params(
# (32, 32, 32),
# (32, 32, 8), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# # *gen_shuffle_params(
# # (64, 64, 16),
# # (32, 32, 8), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# (128, 64, 8),
# (32, 64, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
*gen_shuffle_params(
(128, 64, 8),
(64, 32, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
*gen_shuffle_params(
(64, 64, 8),
(32, 32, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
*gen_shuffle_params(
(32, 64, 16),
(32, 32, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
*gen_shuffle_params(
(64, 32, 16),
(32, 32, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
*gen_shuffle_params(
(32, 32, 32),
(32, 32, 8), ["f32,f32,f32,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# fall back kernels if mat is misaligned for half
# *gen_shuffle_params(
# (64, 128, 16),
# (128, 128, 8),
# (32, 64, 8), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
*gen_shuffle_params(
(32, 64, 32),
(32, 32, 8), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
*gen_shuffle_params(
(32, 32, 32),
(32, 32, 8), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
# *gen_shuffle_params(
# (64, 64, 8),
# (64, 64, 16),
# (32, 32, 8), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
*gen_shuffle_params(
(64, 128, 16),
(32, 64, 8), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
*gen_shuffle_params(
(64, 64, 8),
(32, 32, 8), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2, kernel.GemmAlgo.Simt, None),
]
SHUFFLE_VOLTA_PARAMS: List[GemmAlgoParams] = [
*gen_shuffle_params(
(64, 64, 32),
(32, 32, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
kernel.GemmAlgo.Volta, TensorOpParams((8, 8, 4))),
# *gen_shuffle_params(
# (64, 64, 32),
# (32, 32, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
# kernel.GemmAlgo.Volta, TensorOpParams((8, 8, 4))),
# # *gen_shuffle_params(
# # (128, 128, 32),
# # (64, 64, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
# # kernel.GemmAlgo.Volta, TensorOpParams((8, 8, 4))),
# *gen_shuffle_params(
# (128, 256, 32),
# (64, 64, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
# kernel.GemmAlgo.Volta, TensorOpParams((8, 8, 4))),
# *gen_shuffle_params(
# (256, 128, 32),
# (128, 128, 32),
# (64, 64, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
# kernel.GemmAlgo.Volta, TensorOpParams((8, 8, 4))),
# *gen_shuffle_params(
# (128, 64, 32),
# (64, 32, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
# kernel.GemmAlgo.Volta, TensorOpParams((8, 8, 4))),
# *gen_shuffle_params(
# (64, 128, 32),
# (32, 64, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
# kernel.GemmAlgo.Volta, TensorOpParams((8, 8, 4))),
*gen_shuffle_params(
(128, 256, 32),
(64, 64, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
kernel.GemmAlgo.Volta, TensorOpParams((8, 8, 4))),
*gen_shuffle_params(
(256, 128, 32),
(64, 64, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
kernel.GemmAlgo.Volta, TensorOpParams((8, 8, 4))),
*gen_shuffle_params(
(128, 64, 32),
(64, 32, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
kernel.GemmAlgo.Volta, TensorOpParams((8, 8, 4))),
*gen_shuffle_params(
(64, 128, 32),
(32, 64, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
kernel.GemmAlgo.Volta, TensorOpParams((8, 8, 4))),
]
# SHUFFLE_VOLTA_PARAMS = []
SHUFFLE_TURING_PARAMS: List[GemmAlgoParams] = [
# *gen_shuffle_params(
# (64, 64, 32),
# (32, 32, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
# kernel.GemmAlgo.Turing, TensorOpParams((16, 8, 8))),
*gen_shuffle_params(
(64, 64, 32),
(32, 32, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
kernel.GemmAlgo.Turing, TensorOpParams((16, 8, 8))),
*gen_shuffle_params(
(128, 128, 32),
(32, 64, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
kernel.GemmAlgo.Turing, TensorOpParams((16, 8, 8))),
# *gen_shuffle_params(
# (128, 128, 32),
# (32, 64, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
# kernel.GemmAlgo.Turing, TensorOpParams((16, 8, 8))),
# # *gen_shuffle_params(
# # (128, 128, 32),
# # (64, 32, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
# # kernel.GemmAlgo.Turing, TensorOpParams((16, 8, 8))),
# *gen_shuffle_params(
# (64, 64, 64),
# (32, 32, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
# kernel.GemmAlgo.Turing, TensorOpParams((16, 8, 8))),
# *gen_shuffle_params(
# (64, 128, 64),
# (32, 64, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
# kernel.GemmAlgo.Turing, TensorOpParams((16, 8, 8))),
# *gen_shuffle_params(
# (128, 256, 32),
# (64, 64, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
# kernel.GemmAlgo.Turing, TensorOpParams((16, 8, 8))),
# *gen_shuffle_params(
# (256, 128, 32),
# (64, 64, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
# kernel.GemmAlgo.Turing, TensorOpParams((16, 8, 8))),
# *gen_shuffle_params(
# (128, 64, 32),
# (64, 32, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
# kernel.GemmAlgo.Turing, TensorOpParams((16, 8, 8))),
# *gen_shuffle_params(
# (64, 128, 32),
# (32, 64, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
# kernel.GemmAlgo.Turing, TensorOpParams((16, 8, 8))),
# *gen_shuffle_params(
# (64, 64, 32), (32, 32, 32), ["s8,s8,s32,s32,s32"],
# 2, kernel.GemmAlgo.Turing, TensorOpParams((8, 8, 16))),
*gen_shuffle_params(
(64, 64, 64),
(32, 32, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
kernel.GemmAlgo.Turing, TensorOpParams((16, 8, 8))),
*gen_shuffle_params(
(64, 128, 64),
(32, 64, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
kernel.GemmAlgo.Turing, TensorOpParams((16, 8, 8))),
*gen_shuffle_params(
(128, 256, 32),
(64, 64, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
kernel.GemmAlgo.Turing, TensorOpParams((16, 8, 8))),
*gen_shuffle_params(
(256, 128, 32),
(64, 64, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
kernel.GemmAlgo.Turing, TensorOpParams((16, 8, 8))),
*gen_shuffle_params(
(128, 64, 32),
(64, 32, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
kernel.GemmAlgo.Turing, TensorOpParams((16, 8, 8))),
*gen_shuffle_params(
(64, 128, 32),
(32, 64, 32), ["f16,f16,f16,f16,f16", "f16,f16,f16,f32,f32"], 2,
kernel.GemmAlgo.Turing, TensorOpParams((16, 8, 8))),
*gen_shuffle_params(
(64, 64, 32), (32, 32, 32), ["s8,s8,s32,s32,s32"],
2, kernel.GemmAlgo.Turing, TensorOpParams((8, 8, 16))),
*gen_shuffle_params(
(128, 128, 32),
(32, 64, 32), ["s8,s8,s32,s32,s32"], 2,
kernel.GemmAlgo.Turing, TensorOpParams((8, 8, 16))),
# *gen_shuffle_params(
# (128, 128, 32),
# (32, 64, 32), ["s8,s8,s32,s32,s32"], 2,
# kernel.GemmAlgo.Turing, TensorOpParams((8, 8, 16))),
# # *gen_shuffle_params(
# # (128, 128, 32),
# # (64, 32, 32), ["s8,s8,s8,s32,s32", "s8,s8,s32,s32,s32"], 2,
# # kernel.GemmAlgo.Turing, TensorOpParams((8, 8, 16))),
# *gen_shuffle_params(
# (128, 256, 32),
# (64, 64, 32), ["s8,s8,s32,s32,s32"], 2,
# kernel.GemmAlgo.Turing, TensorOpParams((8, 8, 16))),
# *gen_shuffle_params(
# (256, 128, 32),
# (64, 64, 32), ["s8,s8,s32,s32,s32"], 2,
# (64, 32, 32), ["s8,s8,s8,s32,s32", "s8,s8,s32,s32,s32"], 2,
# kernel.GemmAlgo.Turing, TensorOpParams((8, 8, 16))),
# *gen_shuffle_params(
# (128, 64, 32), (64, 32, 32), ["s8,s8,s32,s32,s32"],
# 2, kernel.GemmAlgo.Turing, TensorOpParams((8, 8, 16))),
# *gen_shuffle_params(
# (64, 128, 32), (32, 64, 32), ["s8,s8,s32,s32,s32"],
# 2, kernel.GemmAlgo.Turing, TensorOpParams((8, 8, 16))),
*gen_shuffle_params(
(128, 256, 32),
(64, 64, 32), ["s8,s8,s32,s32,s32"], 2,
kernel.GemmAlgo.Turing, TensorOpParams((8, 8, 16))),
*gen_shuffle_params(
(256, 128, 32),
(64, 64, 32), ["s8,s8,s32,s32,s32"], 2,
kernel.GemmAlgo.Turing, TensorOpParams((8, 8, 16))),
*gen_shuffle_params(
(128, 64, 32), (64, 32, 32), ["s8,s8,s32,s32,s32"],
2, kernel.GemmAlgo.Turing, TensorOpParams((8, 8, 16))),
*gen_shuffle_params(
(64, 128, 32), (32, 64, 32), ["s8,s8,s32,s32,s32"],
2, kernel.GemmAlgo.Turing, TensorOpParams((8, 8, 16))),
]
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment